Populate tree.

36 files changed, 13514 insertions, 0 deletions

diff --git a/src/rt/bigint/bigint.h b/src/rt/bigint/bigint.h
new file mode 100644
index 00000000..b4c48f03
--- /dev/null
+++ b/src/rt/bigint/bigint.h
@@ -0,0 +1,294 @@
+/* bigint.h - include file for bigint package
+**
+** This library lets you do math on arbitrarily large integers.  It's
+** pretty fast - compared with the multi-precision routines in the "bc"
+** calculator program, these routines are between two and twelve times faster,
+** except for division which is maybe half as fast.
+**
+** The calling convention is a little unusual.  There's a basic problem
+** with writing a math library in a language that doesn't do automatic
+** garbage collection - what do you do about intermediate results?
+** You'd like to be able to write code like this:
+**
+**     d = bi_sqrt( bi_add( bi_multiply( x, x ), bi_multiply( y, y ) ) );
+**
+** That works fine when the numbers being passed back and forth are
+** actual values - ints, floats, or even fixed-size structs.  However,
+** when the numbers can be any size, as in this package, then you have
+** to pass them around as pointers to dynamically-allocated objects.
+** Those objects have to get de-allocated after you are done with them.
+** But how do you de-allocate the intermediate results in a complicated
+** multiple-call expression like the above?
+**
+** There are two common solutions to this problem.  One, switch all your
+** code to a language that provides automatic garbage collection, for
+** example Java.  This is a fine idea and I recommend you do it wherever
+** it's feasible.  Two, change your routines to use a calling convention
+** that prevents people from writing multiple-call expressions like that.
+** The resulting code will be somewhat clumsy-looking, but it will work
+** just fine.
+**
+** This package uses a third method, which I haven't seen used anywhere
+** before.  It's simple: each number can be used precisely once, after
+** which it is automatically de-allocated.  This handles the anonymous
+** intermediate values perfectly.  Named values still need to be copied
+** and freed explicitly.  Here's the above example using this convention:
+**
+**     d = bi_sqrt( bi_add(
+**             bi_multiply( bi_copy( x ), bi_copy( x ) ),
+**             bi_multiply( bi_copy( y ), bi_copy( y ) ) ) );
+**     bi_free( x );
+**     bi_free( y );
+**
+** Or, since the package contains a square routine, you could just write:
+**
+**     d = bi_sqrt( bi_add( bi_square( x ), bi_square( y ) ) );
+**
+** This time the named values are only being used once, so you don't
+** have to copy and free them.
+**
+** This really works, however you do have to be very careful when writing
+** your code.  If you leave out a bi_copy() and use a value more than once,
+** you'll get a runtime error about "zero refs" and a SIGFPE.  Run your
+** code in a debugger, get a backtrace to see where the call was, and then
+** eyeball the code there to see where you need to add the bi_copy().
+**
+**
+** Copyright � 2000 by Jef Poskanzer <[email protected]>.
+** All rights reserved.
+**
+** Redistribution and use in source and binary forms, with or without
+** modification, are permitted provided that the following conditions
+** are met:
+** 1. Redistributions of source code must retain the above copyright
+**    notice, this list of conditions and the following disclaimer.
+** 2. Redistributions in binary form must reproduce the above copyright
+**    notice, this list of conditions and the following disclaimer in the
+**    documentation and/or other materials provided with the distribution.
+**
+** THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+** ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+** IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+** ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+** FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+** DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+** OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+** HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+** LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+** OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+** SUCH DAMAGE.
+*/
+
+
+/* Type definition for bigints - it's an opaque type, the real definition
+** is in bigint.c.
+*/
+typedef void* bigint;
+
+
+/* Some convenient pre-initialized numbers.  These are all permanent,
+** so you can use them as many times as you want without calling bi_copy().
+*/
+extern bigint bi_0, bi_1, bi_2, bi_10, bi_m1, bi_maxint, bi_minint;
+
+
+/* Initialize the bigint package.  You must call this when your program
+** starts up.
+*/
+void bi_initialize( void );
+
+/* Shut down the bigint package.  You should call this when your program
+** exits.  It's not actually required, but it does do some consistency
+** checks which help keep your program bug-free, so you really ought
+** to call it.
+*/
+void bi_terminate( void );
+
+/* Run in unsafe mode, skipping most runtime checks.  Slightly faster.
+** Once your code is debugged you can add this call after bi_initialize().
+*/
+void bi_no_check( void );
+
+/* Make a copy of a bigint.  You must call this if you want to use a
+** bigint more than once.  (Or you can make the bigint permanent.)
+** Note that this routine is very cheap - all it actually does is
+** increment a reference counter.
+*/
+bigint bi_copy( bigint bi );
+
+/* Make a bigint permanent, so it doesn't get automatically freed when
+** used as an operand.
+*/
+void bi_permanent( bigint bi );
+
+/* Undo bi_permanent().  The next use will free the bigint. */
+void bi_depermanent( bigint bi );
+
+/* Explicitly free a bigint.  Normally bigints get freed automatically
+** when they are used as an operand.  This routine lets you free one
+** without using it.  If the bigint is permanent, this doesn't do
+** anything, you have to depermanent it first.
+*/
+void bi_free( bigint bi );
+
+/* Compare two bigints.  Returns -1, 0, or 1. */
+int bi_compare( bigint bia, bigint bib );
+
+/* Convert an int to a bigint. */
+bigint int_to_bi( int i );
+
+/* Convert a string to a bigint. */
+bigint str_to_bi( char* str );
+
+/* Convert a bigint to an int.  SIGFPE on overflow. */
+int bi_to_int( bigint bi );
+
+/* Write a bigint to a file. */
+void bi_print( FILE* f, bigint bi );
+
+/* Read a bigint from a file. */
+bigint bi_scan( FILE* f );
+
+
+/* Operations on a bigint and a regular int. */
+
+/* Add an int to a bigint. */
+bigint bi_int_add( bigint bi, int i );
+
+/* Subtract an int from a bigint. */
+bigint bi_int_subtract( bigint bi, int i );
+
+/* Multiply a bigint by an int. */
+bigint bi_int_multiply( bigint bi, int i );
+
+/* Divide a bigint by an int.  SIGFPE on divide-by-zero. */
+bigint bi_int_divide( bigint binumer, int denom );
+
+/* Take the remainder of a bigint by an int, with an int result.
+** SIGFPE if m is zero.
+*/
+int bi_int_rem( bigint bi, int m );
+
+/* Take the modulus of a bigint by an int, with an int result.
+** Note that mod is not rem: mod is always within [0..m), while
+** rem can be negative.  SIGFPE if m is zero or negative.
+*/
+int bi_int_mod( bigint bi, int m );
+
+
+/* Basic operations on two bigints. */
+
+/* Add two bigints. */
+bigint bi_add( bigint bia, bigint bib );
+
+/* Subtract bib from bia. */
+bigint bi_subtract( bigint bia, bigint bib );
+
+/* Multiply two bigints. */
+bigint bi_multiply( bigint bia, bigint bib );
+
+/* Divide one bigint by another.  SIGFPE on divide-by-zero. */
+bigint bi_divide( bigint binumer, bigint bidenom );
+
+/* Binary division of one bigint by another.  SIGFPE on divide-by-zero.
+** This is here just for testing.  It's about five times slower than
+** regular division.
+*/
+bigint bi_binary_divide( bigint binumer, bigint bidenom );
+
+/* Take the remainder of one bigint by another.  SIGFPE if bim is zero. */
+bigint bi_rem( bigint bia, bigint bim );
+
+/* Take the modulus of one bigint by another.  Note that mod is not rem:
+** mod is always within [0..bim), while rem can be negative.  SIGFPE if
+** bim is zero or negative.
+*/
+bigint bi_mod( bigint bia, bigint bim );
+
+
+/* Some less common operations. */
+
+/* Negate a bigint. */
+bigint bi_negate( bigint bi );
+
+/* Absolute value of a bigint. */
+bigint bi_abs( bigint bi );
+
+/* Divide a bigint in half. */
+bigint bi_half( bigint bi );
+
+/* Multiply a bigint by two. */
+bigint bi_double( bigint bi );
+
+/* Square a bigint. */
+bigint bi_square( bigint bi );
+
+/* Raise bi to the power of biexp.  SIGFPE if biexp is negative. */
+bigint bi_power( bigint bi, bigint biexp );
+
+/* Integer square root. */
+bigint bi_sqrt( bigint bi );
+
+/* Factorial. */
+bigint bi_factorial( bigint bi );
+
+
+/* Some predicates. */
+
+/* 1 if the bigint is odd, 0 if it's even. */
+int bi_is_odd( bigint bi );
+
+/* 1 if the bigint is even, 0 if it's odd. */
+int bi_is_even( bigint bi );
+
+/* 1 if the bigint equals zero, 0 if it's nonzero. */
+int bi_is_zero( bigint bi );
+
+/* 1 if the bigint equals one, 0 otherwise. */
+int bi_is_one( bigint bi );
+
+/* 1 if the bigint is less than zero, 0 if it's zero or greater. */
+int bi_is_negative( bigint bi );
+
+
+/* Now we get into the esoteric number-theory stuff used for cryptography. */
+
+/* Modular exponentiation.  Much faster than bi_mod(bi_power(bi,biexp),bim).
+** Also, biexp can be negative.
+*/
+bigint bi_mod_power( bigint bi, bigint biexp, bigint bim );
+
+/* Modular inverse.  mod( bi * modinv(bi), bim ) == 1.  SIGFPE if bi is not
+** relatively prime to bim.
+*/
+bigint bi_mod_inverse( bigint bi, bigint bim );
+
+/* Produce a random number in the half-open interval [0..bi).  You need
+** to have called srandom() before using this.
+*/
+bigint bi_random( bigint bi );
+
+/* Greatest common divisor of two bigints.  Euclid's algorithm. */
+bigint bi_gcd( bigint bim, bigint bin );
+
+/* Greatest common divisor of two bigints, plus the corresponding multipliers.
+** Extended Euclid's algorithm.
+*/
+bigint bi_egcd( bigint bim, bigint bin, bigint* bim_mul, bigint* bin_mul );
+
+/* Least common multiple of two bigints. */
+bigint bi_lcm( bigint bia, bigint bib );
+
+/* The Jacobi symbol.  SIGFPE if bib is even. */
+bigint bi_jacobi( bigint bia, bigint bib );
+
+/* Probabalistic prime checking.  A non-zero return means the probability
+** that bi is prime is at least 1 - 1/2 ^ certainty.
+*/
+int bi_is_probable_prime( bigint bi, int certainty );
+
+/* Random probabilistic prime with the specified number of bits. */
+bigint bi_generate_prime( int bits, int certainty );
+
+/* Number of bits in the number.  The log base 2, approximately. */
+int bi_bits( bigint bi );
diff --git a/src/rt/bigint/bigint_ext.cpp b/src/rt/bigint/bigint_ext.cpp
new file mode 100644
index 00000000..66d79106
--- /dev/null
+++ b/src/rt/bigint/bigint_ext.cpp
@@ -0,0 +1,553 @@
+/* bigint_ext - external portion of large integer package
+**
+** Copyright � 2000 by Jef Poskanzer <[email protected]>.
+** All rights reserved.
+**
+** Redistribution and use in source and binary forms, with or without
+** modification, are permitted provided that the following conditions
+** are met:
+** 1. Redistributions of source code must retain the above copyright
+**    notice, this list of conditions and the following disclaimer.
+** 2. Redistributions in binary form must reproduce the above copyright
+**    notice, this list of conditions and the following disclaimer in the
+**    documentation and/or other materials provided with the distribution.
+**
+** THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+** ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+** IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+** ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+** FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+** DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+** OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+** HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+** LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+** OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+** SUCH DAMAGE.
+*/
+
+#include <sys/types.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <time.h>
+
+#include "bigint.h"
+#include "low_primes.h"
+
+
+bigint bi_0, bi_1, bi_2, bi_10, bi_m1, bi_maxint, bi_minint;
+
+
+/* Forwards. */
+static void print_pos( FILE* f, bigint bi );
+
+
+bigint
+str_to_bi( char* str )
+    {
+    int sign;
+    bigint biR;
+
+    sign = 1;
+    if ( *str == '-' )
+	{
+	sign = -1;
+	++str;
+	}
+    for ( biR = bi_0; *str >= '0' && *str <= '9'; ++str )
+	biR = bi_int_add( bi_int_multiply( biR, 10 ), *str - '0' );
+    if ( sign == -1 )
+	biR = bi_negate( biR );
+    return biR;
+    }
+
+
+void
+bi_print( FILE* f, bigint bi )
+    {
+    if ( bi_is_negative( bi_copy( bi ) ) )
+	{
+	putc( '-', f );
+	bi = bi_negate( bi );
+	}
+    print_pos( f, bi );
+    }
+
+
+bigint
+bi_scan( FILE* f )
+    {
+    int sign;
+    int c;
+    bigint biR;
+
+    sign = 1;
+    c = getc( f );
+    if ( c == '-' )
+	sign = -1;
+    else
+	ungetc( c, f );
+
+    biR = bi_0;
+    for (;;)
+	{
+	c = getc( f );
+	if ( c < '0' || c > '9' )
+	    break;
+	biR = bi_int_add( bi_int_multiply( biR, 10 ), c - '0' );
+	}
+
+    if ( sign == -1 )
+	biR = bi_negate( biR );
+    return biR;
+    }
+
+
+static void
+print_pos( FILE* f, bigint bi )
+    {
+    if ( bi_compare( bi_copy( bi ), bi_10 ) >= 0 )
+	print_pos( f, bi_int_divide( bi_copy( bi ), 10 ) );
+    putc( bi_int_mod( bi, 10 ) + '0', f );
+    }
+
+
+int
+bi_int_mod( bigint bi, int m )
+    {
+    int r;
+
+    if ( m <= 0 )
+	{
+	(void) fprintf( stderr, "bi_int_mod: zero or negative modulus\n" );
+	(void) kill( getpid(), SIGFPE );
+	}
+    r = bi_int_rem( bi, m );
+    if ( r < 0 )
+	r += m;
+    return r;
+    }
+
+
+bigint
+bi_rem( bigint bia, bigint bim )
+    {
+    return bi_subtract(
+	bia, bi_multiply( bi_divide( bi_copy( bia ), bi_copy( bim ) ), bim ) );
+    }
+
+
+bigint
+bi_mod( bigint bia, bigint bim )
+    {
+    bigint biR;
+
+    if ( bi_compare( bi_copy( bim ), bi_0 ) <= 0 )
+	{
+	(void) fprintf( stderr, "bi_mod: zero or negative modulus\n" );
+	(void) kill( getpid(), SIGFPE );
+	}
+    biR = bi_rem( bia, bi_copy( bim ) );
+    if ( bi_is_negative( bi_copy( biR ) ) )
+	biR = bi_add( biR, bim );
+    else
+	bi_free( bim );
+    return biR;
+    }
+
+
+bigint
+bi_square( bigint bi )
+    {
+    bigint biR;
+
+    biR = bi_multiply( bi_copy( bi ), bi_copy( bi ) );
+    bi_free( bi );
+    return biR;
+    }
+
+
+bigint
+bi_power( bigint bi, bigint biexp )
+    {
+    bigint biR;
+
+    if ( bi_is_negative( bi_copy( biexp ) ) )
+	{
+	(void) fprintf( stderr, "bi_power: negative exponent\n" );
+	(void) kill( getpid(), SIGFPE );
+	}
+    biR = bi_1;
+    for (;;)
+	{
+	if ( bi_is_odd( bi_copy( biexp ) ) )
+	    biR = bi_multiply( biR, bi_copy( bi ) );
+	biexp = bi_half( biexp );
+	if ( bi_compare( bi_copy( biexp ), bi_0 ) <= 0 )
+	    break;
+	bi = bi_multiply( bi_copy( bi ), bi );
+	}
+    bi_free( bi );
+    bi_free( biexp );
+    return biR;
+    }
+
+
+bigint
+bi_factorial( bigint bi )
+    {
+    bigint biR;
+
+    biR = bi_1;
+    while ( bi_compare( bi_copy( bi ), bi_1 ) > 0 )
+	{
+	biR = bi_multiply( biR, bi_copy( bi ) );
+	bi = bi_int_subtract( bi, 1 );
+	}
+    bi_free( bi );
+    return biR;
+    }
+
+
+int
+bi_is_even( bigint bi )
+    {
+    return ! bi_is_odd( bi );
+    }
+
+
+bigint
+bi_mod_power( bigint bi, bigint biexp, bigint bim )
+    {
+    int invert;
+    bigint biR;
+
+    invert = 0;
+    if ( bi_is_negative( bi_copy( biexp ) ) )
+	{
+	biexp = bi_negate( biexp );
+	invert = 1;
+	}
+
+    biR = bi_1;
+    for (;;)
+	{
+	if ( bi_is_odd( bi_copy( biexp ) ) )
+	    biR = bi_mod( bi_multiply( biR, bi_copy( bi ) ), bi_copy( bim ) );
+	biexp = bi_half( biexp );
+	if ( bi_compare( bi_copy( biexp ), bi_0 ) <= 0 )
+	    break;
+	bi = bi_mod( bi_multiply( bi_copy( bi ), bi ), bi_copy( bim ) );
+	}
+    bi_free( bi );
+    bi_free( biexp );
+
+    if ( invert )
+	biR = bi_mod_inverse( biR, bim );
+    else
+	bi_free( bim );
+    return biR;
+    }
+
+
+bigint
+bi_mod_inverse( bigint bi, bigint bim )
+    {
+    bigint gcd, mul0, mul1;
+
+    gcd = bi_egcd( bi_copy( bim ), bi, &mul0, &mul1 );
+
+    /* Did we get gcd == 1? */
+    if ( ! bi_is_one( gcd ) )
+	{
+	(void) fprintf( stderr, "bi_mod_inverse: not relatively prime\n" );
+	(void) kill( getpid(), SIGFPE );
+	}
+
+    bi_free( mul0 );
+    return bi_mod( mul1, bim );
+    }
+
+
+/* Euclid's algorithm. */
+bigint
+bi_gcd( bigint bim, bigint bin )
+    {
+    bigint bit;
+
+    bim = bi_abs( bim );
+    bin = bi_abs( bin );
+    while ( ! bi_is_zero( bi_copy( bin ) ) )
+	{
+	bit = bi_mod( bim, bi_copy( bin ) );
+	bim = bin;
+	bin = bit;
+	}
+    bi_free( bin );
+    return bim;
+    }
+
+
+/* Extended Euclidean algorithm. */
+bigint
+bi_egcd( bigint bim, bigint bin, bigint* bim_mul, bigint* bin_mul )
+    {
+    bigint a0, b0, c0, a1, b1, c1, q, t;
+
+    if ( bi_is_negative( bi_copy( bim ) ) )
+	{
+	bigint biR;
+
+	biR = bi_egcd( bi_negate( bim ), bin, &t, bin_mul );
+	*bim_mul = bi_negate( t );
+	return biR;
+	}
+    if ( bi_is_negative( bi_copy( bin ) ) )
+	{
+	bigint biR;
+
+	biR = bi_egcd( bim, bi_negate( bin ), bim_mul, &t );
+	*bin_mul = bi_negate( t );
+	return biR;
+	}
+
+    a0 = bi_1;  b0 = bi_0;  c0 = bim;
+    a1 = bi_0;  b1 = bi_1;  c1 = bin;
+
+    while ( ! bi_is_zero( bi_copy( c1 ) ) )
+	{
+	q = bi_divide( bi_copy( c0 ), bi_copy( c1 ) );
+	t = a0;
+	a0 = bi_copy( a1 );
+	a1 = bi_subtract( t, bi_multiply( bi_copy( q ), a1 ) );
+	t = b0;
+	b0 = bi_copy( b1 );
+	b1 = bi_subtract( t, bi_multiply( bi_copy( q ), b1 ) );
+	t = c0;
+	c0 = bi_copy( c1 );
+	c1 = bi_subtract( t, bi_multiply( bi_copy( q ), c1 ) );
+	bi_free( q );
+	}
+
+    bi_free( a1 );
+    bi_free( b1 );
+    bi_free( c1 );
+    *bim_mul = a0;
+    *bin_mul = b0;
+    return c0;
+    }
+
+
+bigint
+bi_lcm( bigint bia, bigint bib )
+    {
+    bigint biR;
+
+    biR = bi_divide(
+	bi_multiply( bi_copy( bia ), bi_copy( bib ) ),
+	bi_gcd( bi_copy( bia ), bi_copy( bib ) ) );
+    bi_free( bia );
+    bi_free( bib );
+    return biR;
+    }
+
+
+/* The Jacobi symbol. */
+bigint
+bi_jacobi( bigint bia, bigint bib )
+    {
+    bigint biR;
+
+    if ( bi_is_even( bi_copy( bib ) ) )
+	{
+	(void) fprintf( stderr, "bi_jacobi: don't know how to compute Jacobi(n, even)\n" );
+	(void) kill( getpid(), SIGFPE );
+	}
+
+    if ( bi_compare( bi_copy( bia ), bi_copy( bib ) ) >= 0 )
+	return bi_jacobi( bi_mod( bia, bi_copy( bib ) ), bib );
+
+    if ( bi_is_zero( bi_copy( bia ) ) || bi_is_one( bi_copy( bia ) ) )
+	{
+	bi_free( bib );
+	return bia;
+	}
+
+    if ( bi_compare( bi_copy( bia ), bi_2 ) == 0 )
+	{
+	bi_free( bia );
+	switch ( bi_int_mod( bib, 8 ) )
+	    {
+	    case 1: case 7:
+	    return bi_1;
+	    case 3: case 5:
+	    return bi_m1;
+	    }
+	}
+
+    if ( bi_is_even( bi_copy( bia ) ) )
+	{
+	biR = bi_multiply(
+	    bi_jacobi( bi_2, bi_copy( bib ) ),
+	    bi_jacobi( bi_half( bia ), bi_copy( bib ) ) );
+	bi_free( bib );
+	return biR;
+	}
+
+    if ( bi_int_mod( bi_copy( bia ), 4 ) == 3 &&
+         bi_int_mod( bi_copy( bib ), 4 ) == 3 )
+	return bi_negate( bi_jacobi( bib, bia ) );
+    else
+	return bi_jacobi( bib, bia );
+    }
+
+
+/* Probabalistic prime checking. */
+int
+bi_is_probable_prime( bigint bi, int certainty )
+    {
+    int i, p;
+    bigint bim1;
+
+    /* First do trial division by a list of small primes.  This eliminates
+    ** many candidates.
+    */
+    for ( i = 0; i < sizeof(low_primes)/sizeof(*low_primes); ++i )
+	{
+	p = low_primes[i];
+	switch ( bi_compare( int_to_bi( p ), bi_copy( bi ) ) )
+	    {
+	    case 0:
+	    bi_free( bi );
+	    return 1;
+	    case 1:
+	    bi_free( bi );
+	    return 0;
+	    }
+	if ( bi_int_mod( bi_copy( bi ), p ) == 0 )
+	    {
+	    bi_free( bi );
+	    return 0;
+	    }
+	}
+
+    /* Now do the probabilistic tests. */
+    bim1 = bi_int_subtract( bi_copy( bi ), 1 );
+    for ( i = 0; i < certainty; ++i )
+	{
+	bigint a, j, jac;
+
+	/* Pick random test number. */
+	a = bi_random( bi_copy( bi ) );
+
+	/* Decide whether to run the Fermat test or the Solovay-Strassen
+	** test.  The Fermat test is fast but lets some composite numbers
+	** through.  Solovay-Strassen runs slower but is more certain.
+	** So the compromise here is we run the Fermat test a couple of
+	** times to quickly reject most composite numbers, and then do
+	** the rest of the iterations with Solovay-Strassen so nothing
+	** slips through.
+	*/
+	if ( i < 2 && certainty >= 5 )
+	    {
+	    /* Fermat test.  Note that this is not state of the art.  There's a
+	    ** class of numbers called Carmichael numbers which are composite
+	    ** but look prime to this test - it lets them slip through no
+	    ** matter how many reps you run.  However, it's nice and fast so
+	    ** we run it anyway to help quickly reject most of the composites.
+	    */
+	    if ( ! bi_is_one( bi_mod_power( bi_copy( a ), bi_copy( bim1 ), bi_copy( bi ) ) ) )
+		{
+		bi_free( bi );
+		bi_free( bim1 );
+		bi_free( a );
+		return 0;
+		}
+	    }
+	else
+	    {
+	    /* GCD test.  This rarely hits, but we need it for Solovay-Strassen. */
+	    if ( ! bi_is_one( bi_gcd( bi_copy( bi ), bi_copy( a ) ) ) )
+		{
+		bi_free( bi );
+		bi_free( bim1 );
+		bi_free( a );
+		return 0;
+		}
+
+	    /* Solovay-Strassen test.  First compute pseudo Jacobi. */
+	    j = bi_mod_power(
+		    bi_copy( a ), bi_half( bi_copy( bim1 ) ), bi_copy( bi ) );
+	    if ( bi_compare( bi_copy( j ), bi_copy( bim1 ) ) == 0 )
+		{
+		bi_free( j );
+		j = bi_m1;
+		}
+
+	    /* Now compute real Jacobi. */
+	    jac = bi_jacobi( bi_copy( a ), bi_copy( bi ) );
+
+	    /* If they're not equal, the number is definitely composite. */
+	    if ( bi_compare( j, jac ) != 0 )
+		{
+		bi_free( bi );
+		bi_free( bim1 );
+		bi_free( a );
+		return 0;
+		}
+	    }
+
+	bi_free( a );
+	}
+
+    bi_free( bim1 );
+
+    bi_free( bi );
+    return 1;
+    }
+
+
+bigint
+bi_generate_prime( int bits, int certainty )
+    {
+    bigint bimo2, bip;
+    int i, inc = 0;
+
+    bimo2 = bi_power( bi_2, int_to_bi( bits - 1 ) );
+    for (;;)
+	{
+	bip = bi_add( bi_random( bi_copy( bimo2 ) ), bi_copy( bimo2 ) );
+	/* By shoving the candidate numbers up to the next highest multiple
+	** of six plus or minus one, we pre-eliminate all multiples of
+	** two and/or three.
+	*/
+	switch ( bi_int_mod( bi_copy( bip ), 6 ) )
+	    {
+	    case 0: inc = 4; bip = bi_int_add( bip, 1 ); break;
+	    case 1: inc = 4;                             break;
+	    case 2: inc = 2; bip = bi_int_add( bip, 3 ); break;
+	    case 3: inc = 2; bip = bi_int_add( bip, 2 ); break;
+	    case 4: inc = 2; bip = bi_int_add( bip, 1 ); break;
+	    case 5: inc = 2;                             break;
+	    }
+	/* Starting from the generated random number, check a bunch of
+	** numbers in sequence.  This is just to avoid calls to bi_random(),
+	** which is more expensive than a simple add.
+	*/
+	for ( i = 0; i < 1000; ++i )	/* arbitrary */
+	    {
+	    if ( bi_is_probable_prime( bi_copy( bip ), certainty ) )
+		{
+		bi_free( bimo2 );
+		return bip;
+		}
+	    bip = bi_int_add( bip, inc );
+	    inc = 6 - inc;
+	    }
+	/* We ran through the whole sequence and didn't find a prime.
+	** Shrug, just try a different random starting point.
+	*/
+	bi_free( bip );
+	}
+    }
diff --git a/src/rt/bigint/bigint_int.cpp b/src/rt/bigint/bigint_int.cpp
new file mode 100644
index 00000000..194ddcb5
--- /dev/null
+++ b/src/rt/bigint/bigint_int.cpp
@@ -0,0 +1,1428 @@
+/* bigint - internal portion of large integer package
+**
+** Copyright � 2000 by Jef Poskanzer <[email protected]>.
+** All rights reserved.
+**
+** Redistribution and use in source and binary forms, with or without
+** modification, are permitted provided that the following conditions
+** are met:
+** 1. Redistributions of source code must retain the above copyright
+**    notice, this list of conditions and the following disclaimer.
+** 2. Redistributions in binary form must reproduce the above copyright
+**    notice, this list of conditions and the following disclaimer in the
+**    documentation and/or other materials provided with the distribution.
+**
+** THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+** ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+** IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+** ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+** FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+** DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+** OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+** HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+** LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+** OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+** SUCH DAMAGE.
+*/
+
+#include <sys/types.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <time.h>
+
+#include "bigint.h"
+
+#define max(a,b) ((a)>(b)?(a):(b))
+#define min(a,b) ((a)<(b)?(a):(b))
+
+/* MAXINT and MININT extracted from <values.h>, which gives a warning
+** message if included.
+*/
+#define BITSPERBYTE 8
+#define BITS(type)  (BITSPERBYTE * (int)sizeof(type))
+#define INTBITS     BITS(int)
+#define MININT      (1 << (INTBITS - 1))
+#define MAXINT      (~MININT)
+
+
+/* The package represents arbitrary-precision integers as a sign and a sum
+** of components multiplied by successive powers of the basic radix, i.e.:
+**
+**   sign * ( comp0 + comp1 * radix + comp2 * radix^2 + comp3 * radix^3 )
+**
+** To make good use of the computer's word size, the radix is chosen
+** to be a power of two.  It could be chosen to be the full word size,
+** however this would require a lot of finagling in the middle of the
+** algorithms to get the inter-word overflows right.  That would slow things
+** down.  Instead, the radix is chosen to be *half* the actual word size.
+** With just a little care, this means the words can hold all intermediate
+** values, and the overflows can be handled all at once at the end, in a
+** normalization step.  This simplifies the coding enormously, and is probably
+** somewhat faster to run.  The cost is that numbers use twice as much
+** storage as they would with the most efficient representation, but storage
+** is cheap.
+**
+** A few more notes on the representation:
+**
+**  - The sign is always 1 or -1, never 0.  The number 0 is represented
+**    with a sign of 1.
+**  - The components are signed numbers, to allow for negative intermediate
+**    values.  After normalization, all components are >= 0 and the sign is
+**    updated.
+*/
+
+/* Type definition for bigints. */
+typedef int64_t comp;	/* should be the largest signed int type you have */
+struct _real_bigint {
+    int refs;
+    struct _real_bigint* next;
+    int num_comps, max_comps;
+    int sign;
+    comp* comps;
+    };
+typedef struct _real_bigint* real_bigint;
+
+
+#undef DUMP
+
+
+#define PERMANENT 123456789
+
+static comp bi_radix, bi_radix_o2;
+static int bi_radix_sqrt, bi_comp_bits;
+
+static real_bigint active_list, free_list;
+static int active_count, free_count;
+static int check_level;
+
+
+/* Forwards. */
+static bigint regular_multiply( real_bigint bia, real_bigint bib );
+static bigint multi_divide( bigint binumer, real_bigint bidenom );
+static bigint multi_divide2( bigint binumer, real_bigint bidenom );
+static void more_comps( real_bigint bi, int n );
+static real_bigint alloc( int num_comps );
+static real_bigint clone( real_bigint bi );
+static void normalize( real_bigint bi );
+static void check( real_bigint bi );
+static void double_check( void );
+static void triple_check( void );
+#ifdef DUMP
+static void dump( char* str, bigint bi );
+#endif /* DUMP */
+static int csqrt( comp c );
+static int cbits( comp c );
+
+
+void
+bi_initialize( void )
+    {
+    /* Set the radix.  This does not actually have to be a power of
+    ** two, that's just the most efficient value.  It does have to
+    ** be even for bi_half() to work.
+    */
+    bi_radix = 1;
+    bi_radix <<= BITS(comp) / 2 - 1;
+
+    /* Halve the radix.  Only used by bi_half(). */
+    bi_radix_o2 = bi_radix >> 1;
+
+    /* Take the square root of the radix.  Only used by bi_divide(). */
+    bi_radix_sqrt = csqrt( bi_radix );
+
+    /* Figure out how many bits in a component.  Only used by bi_bits(). */
+    bi_comp_bits = cbits( bi_radix - 1 );
+
+    /* Init various globals. */
+    active_list = (real_bigint) 0;
+    active_count = 0;
+    free_list = (real_bigint) 0;
+    free_count = 0;
+
+    /* This can be 0 through 3. */
+    check_level = 3;
+
+    /* Set up some convenient bigints. */
+    bi_0 = int_to_bi( 0 ); bi_permanent( bi_0 );
+    bi_1 = int_to_bi( 1 ); bi_permanent( bi_1 );
+    bi_2 = int_to_bi( 2 ); bi_permanent( bi_2 );
+    bi_10 = int_to_bi( 10 ); bi_permanent( bi_10 );
+    bi_m1 = int_to_bi( -1 ); bi_permanent( bi_m1 );
+    bi_maxint = int_to_bi( MAXINT ); bi_permanent( bi_maxint );
+    bi_minint = int_to_bi( MININT ); bi_permanent( bi_minint );
+    }
+
+
+void
+bi_terminate( void )
+    {
+    real_bigint p, pn;
+
+    bi_depermanent( bi_0 ); bi_free( bi_0 );
+    bi_depermanent( bi_1 ); bi_free( bi_1 );
+    bi_depermanent( bi_2 ); bi_free( bi_2 );
+    bi_depermanent( bi_10 ); bi_free( bi_10 );
+    bi_depermanent( bi_m1 ); bi_free( bi_m1 );
+    bi_depermanent( bi_maxint ); bi_free( bi_maxint );
+    bi_depermanent( bi_minint ); bi_free( bi_minint );
+
+    if ( active_count != 0 )
+	(void) fprintf(
+	    stderr, "bi_terminate: there were %d un-freed bigints\n",
+	    active_count );
+    if ( check_level >= 2 )
+	double_check();
+    if ( check_level >= 3 )
+	{
+	triple_check();
+	for ( p = active_list; p != (bigint) 0; p = pn )
+	    {
+	    pn = p->next;
+	    free( p->comps );
+	    free( p );
+	    }
+	}
+    for ( p = free_list; p != (bigint) 0; p = pn )
+	{
+	pn = p->next;
+	free( p->comps );
+	free( p );
+	}
+    }
+
+
+void
+bi_no_check( void )
+    {
+    check_level = 0;
+    }
+
+
+bigint
+bi_copy( bigint obi )
+    {
+    real_bigint bi = (real_bigint) obi;
+
+    check( bi );
+    if ( bi->refs != PERMANENT )
+	++bi->refs;
+    return bi;
+    }
+
+
+void
+bi_permanent( bigint obi )
+    {
+    real_bigint bi = (real_bigint) obi;
+
+    check( bi );
+    if ( check_level >= 1 && bi->refs != 1 )
+	{
+	(void) fprintf( stderr, "bi_permanent: refs was not 1\n" );
+	(void) kill( getpid(), SIGFPE );
+	}
+    bi->refs = PERMANENT;
+    }
+
+
+void
+bi_depermanent( bigint obi )
+    {
+    real_bigint bi = (real_bigint) obi;
+
+    check( bi );
+    if ( check_level >= 1 && bi->refs != PERMANENT )
+	{
+	(void) fprintf( stderr, "bi_depermanent: bigint was not permanent\n" );
+	(void) kill( getpid(), SIGFPE );
+	}
+    bi->refs = 1;
+    }
+
+
+void
+bi_free( bigint obi )
+    {
+    real_bigint bi = (real_bigint) obi;
+
+    check( bi );
+    if ( bi->refs == PERMANENT )
+	return;
+    --bi->refs;
+    if ( bi->refs > 0 )
+	return;
+    if ( check_level >= 3 )
+	{
+	/* The active list only gets maintained at check levels 3 or higher. */
+	real_bigint* nextP;
+	for ( nextP = &active_list; *nextP != (real_bigint) 0; nextP = &((*nextP)->next) )
+	    if ( *nextP == bi )
+		{
+		*nextP = bi->next;
+		break;
+		}
+	}
+    --active_count;
+    bi->next = free_list;
+    free_list = bi;
+    ++free_count;
+    if ( check_level >= 1 && active_count < 0 )
+	{
+	(void) fprintf( stderr,
+	    "bi_free: active_count went negative - double-freed bigint?\n" );
+	(void) kill( getpid(), SIGFPE );
+	}
+    }
+
+
+int
+bi_compare( bigint obia, bigint obib )
+    {
+    real_bigint bia = (real_bigint) obia;
+    real_bigint bib = (real_bigint) obib;
+    int r, c;
+
+    check( bia );
+    check( bib );
+
+    /* First check for pointer equality. */
+    if ( bia == bib )
+	r = 0;
+    else
+	{
+	/* Compare signs. */
+	if ( bia->sign > bib->sign )
+	    r = 1;
+	else if ( bia->sign < bib->sign )
+	    r = -1;
+	/* Signs are the same.  Check the number of components. */
+	else if ( bia->num_comps > bib->num_comps )
+	    r = bia->sign;
+	else if ( bia->num_comps < bib->num_comps )
+	    r = -bia->sign;
+	else
+	    {
+	    /* Same number of components.  Compare starting from the high end
+	    ** and working down.
+	    */
+	    r = 0;	/* if we complete the loop, the numbers are equal */
+	    for ( c = bia->num_comps - 1; c >= 0; --c )
+		{
+		if ( bia->comps[c] > bib->comps[c] )
+		    { r = bia->sign; break; }
+		else if ( bia->comps[c] < bib->comps[c] )
+		    { r = -bia->sign; break; }
+		}
+	    }
+	}
+
+    bi_free( bia );
+    bi_free( bib );
+    return r;
+    }
+
+
+bigint
+int_to_bi( int i )
+    {
+    real_bigint biR;
+
+    biR = alloc( 1 );
+    biR->sign = 1;
+    biR->comps[0] = i;
+    normalize( biR );
+    check( biR );
+    return biR;
+    }
+
+
+int
+bi_to_int( bigint obi )
+    {
+    real_bigint bi = (real_bigint) obi;
+    comp v, m;
+    int c, r;
+
+    check( bi );
+    if ( bi_compare( bi_copy( bi ), bi_maxint ) > 0 ||
+	 bi_compare( bi_copy( bi ), bi_minint ) < 0 )
+	{
+	(void) fprintf( stderr, "bi_to_int: overflow\n" );
+	(void) kill( getpid(), SIGFPE );
+	}
+    v = 0;
+    m = 1;
+    for ( c = 0; c < bi->num_comps; ++c )
+	{
+	v += bi->comps[c] * m;
+	m *= bi_radix;
+	}
+    r = (int) ( bi->sign * v );
+    bi_free( bi );
+    return r;
+    }
+
+
+bigint
+bi_int_add( bigint obi, int i )
+    {
+    real_bigint bi = (real_bigint) obi;
+    real_bigint biR;
+
+    check( bi );
+    biR = clone( bi );
+    if ( biR->sign == 1 )
+	biR->comps[0] += i;
+    else
+	biR->comps[0] -= i;
+    normalize( biR );
+    check( biR );
+    return biR;
+    }
+
+
+bigint
+bi_int_subtract( bigint obi, int i )
+    {
+    real_bigint bi = (real_bigint) obi;
+    real_bigint biR;
+
+    check( bi );
+    biR = clone( bi );
+    if ( biR->sign == 1 )
+	biR->comps[0] -= i;
+    else
+	biR->comps[0] += i;
+    normalize( biR );
+    check( biR );
+    return biR;
+    }
+
+
+bigint
+bi_int_multiply( bigint obi, int i )
+    {
+    real_bigint bi = (real_bigint) obi;
+    real_bigint biR;
+    int c;
+
+    check( bi );
+    biR = clone( bi );
+    if ( i < 0 )
+	{
+	i = -i;
+	biR->sign = -biR->sign;
+	}
+    for ( c = 0; c < biR->num_comps; ++c )
+	biR->comps[c] *= i;
+    normalize( biR );
+    check( biR );
+    return biR;
+    }
+
+
+bigint
+bi_int_divide( bigint obinumer, int denom )
+    {
+    real_bigint binumer = (real_bigint) obinumer;
+    real_bigint biR;
+    int c;
+    comp r;
+
+    check( binumer );
+    if ( denom == 0 )
+	{
+	(void) fprintf( stderr, "bi_int_divide: divide by zero\n" );
+	(void) kill( getpid(), SIGFPE );
+	}
+    biR = clone( binumer );
+    if ( denom < 0 )
+	{
+	denom = -denom;
+	biR->sign = -biR->sign;
+	}
+    r = 0;
+    for ( c = biR->num_comps - 1; c >= 0; --c )
+	{
+	r = r * bi_radix + biR->comps[c];
+	biR->comps[c] = r / denom;
+	r = r % denom;
+	}
+    normalize( biR );
+    check( biR );
+    return biR;
+    }
+
+
+int
+bi_int_rem( bigint obi, int m )
+    {
+    real_bigint bi = (real_bigint) obi;
+    comp rad_r, r;
+    int  c;
+
+    check( bi );
+    if ( m == 0 )
+	{
+	(void) fprintf( stderr, "bi_int_rem: divide by zero\n" );
+	(void) kill( getpid(), SIGFPE );
+	}
+    if ( m < 0 )
+	m = -m;
+    rad_r = 1;
+    r = 0;
+    for ( c = 0; c < bi->num_comps; ++c )
+	{
+	r = ( r + bi->comps[c] * rad_r ) % m;
+	rad_r = ( rad_r * bi_radix ) % m;
+	}
+    if ( bi->sign < 1 )
+	r = -r;
+    bi_free( bi );
+    return (int) r;
+    }
+
+
+bigint
+bi_add( bigint obia, bigint obib )
+    {
+    real_bigint bia = (real_bigint) obia;
+    real_bigint bib = (real_bigint) obib;
+    real_bigint biR;
+    int c;
+
+    check( bia );
+    check( bib );
+    biR = clone( bia );
+    more_comps( biR, max( biR->num_comps, bib->num_comps ) );
+    for ( c = 0; c < bib->num_comps; ++c )
+	if ( biR->sign == bib->sign )
+	    biR->comps[c] += bib->comps[c];
+	else
+	    biR->comps[c] -= bib->comps[c];
+    bi_free( bib );
+    normalize( biR );
+    check( biR );
+    return biR;
+    }
+
+
+bigint
+bi_subtract( bigint obia, bigint obib )
+    {
+    real_bigint bia = (real_bigint) obia;
+    real_bigint bib = (real_bigint) obib;
+    real_bigint biR;
+    int c;
+
+    check( bia );
+    check( bib );
+    biR = clone( bia );
+    more_comps( biR, max( biR->num_comps, bib->num_comps ) );
+    for ( c = 0; c < bib->num_comps; ++c )
+	if ( biR->sign == bib->sign )
+	    biR->comps[c] -= bib->comps[c];
+	else
+	    biR->comps[c] += bib->comps[c];
+    bi_free( bib );
+    normalize( biR );
+    check( biR );
+    return biR;
+    }
+
+
+/* Karatsuba multiplication.  This is supposedly O(n^1.59), better than
+** regular multiplication for large n.  The define below sets the crossover
+** point - below that we use regular multiplication, above it we
+** use Karatsuba.  Note that Karatsuba is a recursive algorithm, so
+** all Karatsuba calls involve regular multiplications as the base
+** steps.
+*/
+#define KARATSUBA_THRESH 12
+bigint
+bi_multiply( bigint obia, bigint obib )
+    {
+    real_bigint bia = (real_bigint) obia;
+    real_bigint bib = (real_bigint) obib;
+
+    check( bia );
+    check( bib );
+    if ( min( bia->num_comps, bib->num_comps ) < KARATSUBA_THRESH )
+	return regular_multiply( bia, bib );
+    else
+	{
+	/* The factors are large enough that Karatsuba multiplication
+	** is a win.  The basic idea here is you break each factor up
+	** into two parts, like so:
+	**     i * r^n + j        k * r^n + l
+	** r is the radix we're representing numbers with, so this
+	** breaking up just means shuffling components around, no
+	** math required.  With regular multiplication the product
+	** would be:
+	**     ik * r^(n*2) + ( il + jk ) * r^n + jl
+	** That's four sub-multiplies and one addition, not counting the
+	** radix-shifting.  With Karatsuba, you instead do:
+	**     ik * r^(n*2) + ( (i+j)(k+l) - ik - jl ) * r^n  + jl
+	** This is only three sub-multiplies.  The number of adds
+	** (and subtracts) increases to four, but those run in linear time
+	** so they are cheap.  The sub-multiplies are accomplished by
+	** recursive calls, eventually reducing to regular multiplication.
+	*/
+	int n, c;
+	real_bigint bi_i, bi_j, bi_k, bi_l;
+	real_bigint bi_ik, bi_mid, bi_jl;
+
+	n = ( max( bia->num_comps, bib->num_comps ) + 1 ) / 2;
+	bi_i = alloc( n );
+	bi_j = alloc( n );
+	bi_k = alloc( n );
+	bi_l = alloc( n );
+	for ( c = 0; c < n; ++c )
+	    {
+	    if ( c + n < bia->num_comps )
+		bi_i->comps[c] = bia->comps[c + n];
+	    else
+		bi_i->comps[c] = 0;
+	    if ( c < bia->num_comps )
+		bi_j->comps[c] = bia->comps[c];
+	    else
+		bi_j->comps[c] = 0;
+	    if ( c + n < bib->num_comps )
+		bi_k->comps[c] = bib->comps[c + n];
+	    else
+		bi_k->comps[c] = 0;
+	    if ( c < bib->num_comps )
+		bi_l->comps[c] = bib->comps[c];
+	    else
+		bi_l->comps[c] = 0;
+	    }
+	bi_i->sign = bi_j->sign = bi_k->sign = bi_l->sign = 1;
+	normalize( bi_i );
+	normalize( bi_j );
+	normalize( bi_k );
+	normalize( bi_l );
+	bi_ik = bi_multiply( bi_copy( bi_i ), bi_copy( bi_k ) );
+	bi_jl = bi_multiply( bi_copy( bi_j ), bi_copy( bi_l ) );
+	bi_mid = bi_subtract(
+	    bi_subtract(
+		bi_multiply( bi_add( bi_i, bi_j ), bi_add( bi_k, bi_l ) ),
+		bi_copy( bi_ik ) ),
+	    bi_copy( bi_jl ) );
+	more_comps(
+	    bi_jl, max( bi_mid->num_comps + n, bi_ik->num_comps + n * 2 ) );
+	for ( c = 0; c < bi_mid->num_comps; ++c )
+	    bi_jl->comps[c + n] += bi_mid->comps[c];
+	for ( c = 0; c < bi_ik->num_comps; ++c )
+	    bi_jl->comps[c + n * 2] += bi_ik->comps[c];
+	bi_free( bi_ik );
+	bi_free( bi_mid );
+	bi_jl->sign = bia->sign * bib->sign;
+	bi_free( bia );
+	bi_free( bib );
+	normalize( bi_jl );
+	check( bi_jl );
+	return bi_jl;
+	}
+    }
+
+
+/* Regular O(n^2) multiplication. */
+static bigint
+regular_multiply( real_bigint bia, real_bigint bib )
+    {
+    real_bigint biR;
+    int new_comps, c1, c2;
+
+    check( bia );
+    check( bib );
+    biR = clone( bi_0 );
+    new_comps = bia->num_comps + bib->num_comps;
+    more_comps( biR, new_comps );
+    for ( c1 = 0; c1 < bia->num_comps; ++c1 )
+	{
+	for ( c2 = 0; c2 < bib->num_comps; ++c2 )
+	    biR->comps[c1 + c2] += bia->comps[c1] * bib->comps[c2];
+	/* Normalize after each inner loop to avoid overflowing any
+	** components.  But be sure to reset biR's components count,
+	** in case a previous normalization lowered it.
+	*/
+	biR->num_comps = new_comps;
+	normalize( biR );
+	}
+    check( biR );
+    if ( ! bi_is_zero( bi_copy( biR ) ) )
+	biR->sign = bia->sign * bib->sign;
+    bi_free( bia );
+    bi_free( bib );
+    return biR;
+    }
+
+
+/* The following three routines implement a multi-precision divide method
+** that I haven't seen used anywhere else.  It is not quite as fast as
+** the standard divide method, but it is a lot simpler.  In fact it's
+** about as simple as the binary shift-and-subtract method, which goes
+** about five times slower than this.
+**
+** The method assumes you already have multi-precision multiply and subtract
+** routines, and also a multi-by-single precision divide routine.  The latter
+** is used to generate approximations, which are then checked and corrected
+** using the former.  The result converges to the correct value by about
+** 16 bits per loop.
+*/
+
+/* Public routine to divide two arbitrary numbers. */
+bigint
+bi_divide( bigint binumer, bigint obidenom )
+    {
+    real_bigint bidenom = (real_bigint) obidenom;
+    int sign;
+    bigint biquotient;
+
+    /* Check signs and trivial cases. */
+    sign = 1;
+    switch ( bi_compare( bi_copy( bidenom ), bi_0 ) )
+	{
+	case 0:
+	(void) fprintf( stderr, "bi_divide: divide by zero\n" );
+	(void) kill( getpid(), SIGFPE );
+	case -1:
+	sign *= -1;
+	bidenom = bi_negate( bidenom );
+	break;
+	}
+    switch ( bi_compare( bi_copy( binumer ), bi_0 ) )
+	{
+	case 0:
+	bi_free( binumer );
+	bi_free( bidenom );
+	return bi_0;
+	case -1:
+	sign *= -1;
+	binumer = bi_negate( binumer );
+	break;
+	}
+    switch ( bi_compare( bi_copy( binumer ), bi_copy( bidenom ) ) )
+	{
+	case -1:
+	bi_free( binumer );
+	bi_free( bidenom );
+	return bi_0;
+	case 0:
+	bi_free( binumer );
+	bi_free( bidenom );
+	if ( sign == 1 )
+	    return bi_1;
+	else
+	    return bi_m1;
+	}
+
+    /* Is the denominator small enough to do an int divide? */
+    if ( bidenom->num_comps == 1 )
+	{
+	/* Win! */
+	biquotient = bi_int_divide( binumer, bidenom->comps[0] );
+	bi_free( bidenom );
+	}
+    else
+	{
+	/* No, we have to do a full multi-by-multi divide. */
+	biquotient = multi_divide( binumer, bidenom );
+	}
+
+    if ( sign == -1 )
+	biquotient = bi_negate( biquotient );
+    return biquotient;
+    }
+
+
+/* Divide two multi-precision positive numbers. */
+static bigint
+multi_divide( bigint binumer, real_bigint bidenom )
+    {
+    /* We use a successive approximation method that is kind of like a
+    ** continued fraction.  The basic approximation is to do an int divide
+    ** by the high-order component of the denominator.  Then we correct
+    ** based on the remainder from that.
+    **
+    ** However, if the high-order component is too small, this doesn't
+    ** work well.  In particular, if the high-order component is 1 it
+    ** doesn't work at all.  Easily fixed, though - if the component
+    ** is too small, increase it!
+    */
+    if ( bidenom->comps[bidenom->num_comps-1] < bi_radix_sqrt )
+	{
+	/* We use the square root of the radix as the threshhold here
+	** because that's the largest value guaranteed to not make the
+	** high-order component overflow and become too small again.
+	**
+	** We increase binumer along with bidenom to keep the end result
+	** the same.
+	*/
+	binumer = bi_int_multiply( binumer, bi_radix_sqrt );
+	bidenom = bi_int_multiply( bidenom, bi_radix_sqrt );
+	}
+
+    /* Now start the recursion. */
+    return multi_divide2( binumer, bidenom );
+    }
+
+
+/* Divide two multi-precision positive conditioned numbers. */
+static bigint
+multi_divide2( bigint binumer, real_bigint bidenom )
+    {
+    real_bigint biapprox;
+    bigint birem, biquotient;
+    int c, o;
+
+    /* Figure out the approximate quotient.   Since we're dividing by only
+    ** the top component of the denominator, which is less than or equal to
+    ** the full denominator, the result is guaranteed to be greater than or
+    ** equal to the correct quotient.
+    */
+    o = bidenom->num_comps - 1;
+    biapprox = bi_int_divide( bi_copy( binumer ), bidenom->comps[o] );
+    /* And downshift the result to get the approximate quotient. */
+    for ( c = o; c < biapprox->num_comps; ++c )
+	biapprox->comps[c - o] = biapprox->comps[c];
+    biapprox->num_comps -= o;
+
+    /* Find the remainder from the approximate quotient. */
+    birem = bi_subtract(
+	bi_multiply( bi_copy( biapprox ), bi_copy( bidenom ) ), binumer );
+
+    /* If the remainder is negative, zero, or in fact any value less
+    ** than bidenom, then we have the correct quotient and we're done.
+    */
+    if ( bi_compare( bi_copy( birem ), bi_copy( bidenom ) ) < 0 )
+	{
+	biquotient = biapprox;
+	bi_free( birem );
+	bi_free( bidenom );
+	}
+    else
+	{
+	/* The real quotient is now biapprox - birem / bidenom.  We still
+	** have to do a divide.  However, birem is smaller than binumer,
+	** so the next divide will go faster.  We do the divide by
+	** recursion.  Since this is tail-recursion or close to it, we
+	** could probably re-arrange things and make it a non-recursive
+	** loop, but the overhead of recursion is small and the bookkeeping
+	** is simpler this way.
+	**
+	** Note that since the sub-divide uses the same denominator, it
+	** doesn't have to adjust the values again - the high-order component
+	** will still be good.
+	*/
+	biquotient = bi_subtract( biapprox, multi_divide2( birem, bidenom ) );
+	}
+
+    return biquotient;
+    }
+
+
+/* Binary division - about five times slower than the above. */
+bigint
+bi_binary_divide( bigint binumer, bigint obidenom )
+    {
+    real_bigint bidenom = (real_bigint) obidenom;
+    int sign;
+    bigint biquotient;
+
+    /* Check signs and trivial cases. */
+    sign = 1;
+    switch ( bi_compare( bi_copy( bidenom ), bi_0 ) )
+	{
+	case 0:
+	(void) fprintf( stderr, "bi_divide: divide by zero\n" );
+	(void) kill( getpid(), SIGFPE );
+	case -1:
+	sign *= -1;
+	bidenom = bi_negate( bidenom );
+	break;
+	}
+    switch ( bi_compare( bi_copy( binumer ), bi_0 ) )
+	{
+	case 0:
+	bi_free( binumer );
+	bi_free( bidenom );
+	return bi_0;
+	case -1:
+	sign *= -1;
+	binumer = bi_negate( binumer );
+	break;
+	}
+    switch ( bi_compare( bi_copy( binumer ), bi_copy( bidenom ) ) )
+	{
+	case -1:
+	bi_free( binumer );
+	bi_free( bidenom );
+	return bi_0;
+	case 0:
+	bi_free( binumer );
+	bi_free( bidenom );
+	if ( sign == 1 )
+	    return bi_1;
+	else
+	    return bi_m1;
+	}
+
+    /* Is the denominator small enough to do an int divide? */
+    if ( bidenom->num_comps == 1 )
+	{
+	/* Win! */
+	biquotient = bi_int_divide( binumer, bidenom->comps[0] );
+	bi_free( bidenom );
+	}
+    else
+	{
+	/* No, we have to do a full multi-by-multi divide. */
+	int num_bits, den_bits, i;
+
+	num_bits = bi_bits( bi_copy( binumer ) );
+	den_bits = bi_bits( bi_copy( bidenom ) );
+	bidenom = bi_multiply( bidenom, bi_power( bi_2, int_to_bi( num_bits - den_bits ) ) );
+	biquotient = bi_0;
+	for ( i = den_bits; i <= num_bits; ++i )
+	    {
+	    biquotient = bi_double( biquotient );
+	    if ( bi_compare( bi_copy( binumer ), bi_copy( bidenom ) ) >= 0 )
+		{
+		biquotient = bi_int_add( biquotient, 1 );
+		binumer = bi_subtract( binumer, bi_copy( bidenom ) );
+		}
+	    bidenom = bi_half( bidenom );
+	    }
+	bi_free( binumer );
+	bi_free( bidenom );
+	}
+
+    if ( sign == -1 )
+	biquotient = bi_negate( biquotient );
+    return biquotient;
+    }
+
+
+bigint
+bi_negate( bigint obi )
+    {
+    real_bigint bi = (real_bigint) obi;
+    real_bigint biR;
+
+    check( bi );
+    biR = clone( bi );
+    biR->sign = -biR->sign;
+    check( biR );
+    return biR;
+    }
+
+
+bigint
+bi_abs( bigint obi )
+    {
+    real_bigint bi = (real_bigint) obi;
+    real_bigint biR;
+
+    check( bi );
+    biR = clone( bi );
+    biR->sign = 1;
+    check( biR );
+    return biR;
+    }
+
+
+bigint
+bi_half( bigint obi )
+    {
+    real_bigint bi = (real_bigint) obi;
+    real_bigint biR;
+    int c;
+
+    check( bi );
+    /* This depends on the radix being even. */
+    biR = clone( bi );
+    for ( c = 0; c < biR->num_comps; ++c )
+	{
+	if ( biR->comps[c] & 1 )
+	    if ( c > 0 )
+		biR->comps[c - 1] += bi_radix_o2;
+	biR->comps[c] = biR->comps[c] >> 1;
+	}
+    /* Avoid normalization. */
+    if ( biR->num_comps > 1 && biR->comps[biR->num_comps-1] == 0 )
+	--biR->num_comps;
+    check( biR );
+    return biR;
+    }
+
+
+bigint
+bi_double( bigint obi )
+    {
+    real_bigint bi = (real_bigint) obi;
+    real_bigint biR;
+    int c;
+
+    check( bi );
+    biR = clone( bi );
+    for ( c = biR->num_comps - 1; c >= 0; --c )
+	{
+	biR->comps[c] = biR->comps[c] << 1;
+	if ( biR->comps[c] >= bi_radix )
+	    {
+	    if ( c + 1 >= biR->num_comps )
+		more_comps( biR, biR->num_comps + 1 );
+	    biR->comps[c] -= bi_radix;
+	    biR->comps[c + 1] += 1;
+	    }
+	}
+    check( biR );
+    return biR;
+    }
+
+
+/* Find integer square root by Newton's method. */
+bigint
+bi_sqrt( bigint obi )
+    {
+    real_bigint bi = (real_bigint) obi;
+    bigint biR, biR2, bidiff;
+
+    switch ( bi_compare( bi_copy( bi ), bi_0 ) )
+	{
+	case -1:
+	(void) fprintf( stderr, "bi_sqrt: imaginary result\n" );
+	(void) kill( getpid(), SIGFPE );
+	case 0:
+	return bi;
+	}
+    if ( bi_is_one( bi_copy( bi ) ) )
+	return bi;
+
+    /* Newton's method converges reasonably fast, but it helps to have
+    ** a good initial guess.  We can make a *very* good initial guess
+    ** by taking the square root of the top component times the square
+    ** root of the radix part.  Both of those are easy to compute.
+    */
+    biR = bi_int_multiply(
+	bi_power( int_to_bi( bi_radix_sqrt ), int_to_bi( bi->num_comps - 1 ) ),
+	csqrt( bi->comps[bi->num_comps - 1] ) );
+
+    /* Now do the Newton loop until we have the answer. */
+    for (;;)
+	{
+	biR2 = bi_divide( bi_copy( bi ), bi_copy( biR ) );
+	bidiff = bi_subtract( bi_copy( biR ), bi_copy( biR2 ) );
+	if ( bi_is_zero( bi_copy( bidiff ) ) ||
+	     bi_compare( bi_copy( bidiff ), bi_m1 ) == 0 )
+	    {
+	    bi_free( bi );
+	    bi_free( bidiff );
+	    bi_free( biR2 );
+	    return biR;
+	    }
+	if ( bi_is_one( bi_copy( bidiff ) ) )
+	    {
+	    bi_free( bi );
+	    bi_free( bidiff );
+	    bi_free( biR );
+	    return biR2;
+	    }
+	bi_free( bidiff );
+	biR = bi_half( bi_add( biR, biR2 ) );
+	}
+    }
+
+
+int
+bi_is_odd( bigint obi )
+    {
+    real_bigint bi = (real_bigint) obi;
+    int r;
+
+    check( bi );
+    r = bi->comps[0] & 1;
+    bi_free( bi );
+    return r;
+    }
+
+
+int
+bi_is_zero( bigint obi )
+    {
+    real_bigint bi = (real_bigint) obi;
+    int r;
+
+    check( bi );
+    r = ( bi->sign == 1 && bi->num_comps == 1 && bi->comps[0] == 0 );
+    bi_free( bi );
+    return r;
+    }
+
+
+int
+bi_is_one( bigint obi )
+    {
+    real_bigint bi = (real_bigint) obi;
+    int r;
+
+    check( bi );
+    r = ( bi->sign == 1 && bi->num_comps == 1 && bi->comps[0] == 1 );
+    bi_free( bi );
+    return r;
+    }
+
+
+int
+bi_is_negative( bigint obi )
+    {
+    real_bigint bi = (real_bigint) obi;
+    int r;
+
+    check( bi );
+    r = ( bi->sign == -1 );
+    bi_free( bi );
+    return r;
+    }
+
+
+bigint
+bi_random( bigint bi )
+    {
+    real_bigint biR;
+    int c;
+
+    biR = bi_multiply( bi_copy( bi ), bi_copy( bi ) );
+    for ( c = 0; c < biR->num_comps; ++c )
+	biR->comps[c] = random();
+    normalize( biR );
+    biR = bi_mod( biR, bi );
+    return biR;
+    }
+
+
+int
+bi_bits( bigint obi )
+    {
+    real_bigint bi = (real_bigint) obi;
+    int bits;
+
+    bits =
+	bi_comp_bits * ( bi->num_comps - 1 ) +
+	cbits( bi->comps[bi->num_comps - 1] );
+    bi_free( bi );
+    return bits;
+    }
+
+
+/* Allocate and zero more components.  Does not consume bi, of course. */
+static void
+more_comps( real_bigint bi, int n )
+    {
+    if ( n > bi->max_comps )
+	{
+	bi->max_comps = max( bi->max_comps * 2, n );
+	bi->comps = (comp*) realloc(
+	    (void*) bi->comps, bi->max_comps * sizeof(comp) );
+	if ( bi->comps == (comp*) 0 )
+	    {
+	    (void) fprintf( stderr, "out of memory\n" );
+	    exit( 1 );
+	    }
+	}
+    for ( ; bi->num_comps < n; ++bi->num_comps )
+	bi->comps[bi->num_comps] = 0;
+    }
+
+
+/* Make a new empty bigint.  Fills in everything except sign and the
+** components.
+*/
+static real_bigint
+alloc( int num_comps )
+    {
+    real_bigint biR;
+
+    /* Can we recycle an old bigint? */
+    if ( free_list != (real_bigint) 0 )
+	{
+	biR = free_list;
+	free_list = biR->next;
+	--free_count;
+	if ( check_level >= 1 && biR->refs != 0 )
+	    {
+	    (void) fprintf( stderr, "alloc: refs was not 0\n" );
+	    (void) kill( getpid(), SIGFPE );
+	    }
+	more_comps( biR, num_comps );
+	}
+    else
+	{
+	/* No free bigints available - create a new one. */
+	biR = (real_bigint) malloc( sizeof(struct _real_bigint) );
+	if ( biR == (real_bigint) 0 )
+	    {
+	    (void) fprintf( stderr, "out of memory\n" );
+	    exit( 1 );
+	    }
+	biR->comps = (comp*) malloc( num_comps * sizeof(comp) );
+	if ( biR->comps == (comp*) 0 )
+	    {
+	    (void) fprintf( stderr, "out of memory\n" );
+	    exit( 1 );
+	    }
+	biR->max_comps = num_comps;
+	}
+    biR->num_comps = num_comps;
+    biR->refs = 1;
+    if ( check_level >= 3 )
+	{
+	/* The active list only gets maintained at check levels 3 or higher. */
+	biR->next = active_list;
+	active_list = biR;
+	}
+    else
+	biR->next = (real_bigint) 0;
+    ++active_count;
+    return biR;
+    }
+
+
+/* Make a modifiable copy of bi.  DOES consume bi. */
+static real_bigint
+clone( real_bigint bi )
+    {
+    real_bigint biR;
+    int c;
+
+    /* Very clever optimization. */
+    if ( bi->refs != PERMANENT && bi->refs == 1 )
+	return bi;
+
+    biR = alloc( bi->num_comps );
+    biR->sign = bi->sign;
+    for ( c = 0; c < bi->num_comps; ++c )
+	biR->comps[c] = bi->comps[c];
+    bi_free( bi );
+    return biR;
+    }
+
+
+/* Put bi into normal form.  Does not consume bi, of course.
+**
+** Normal form is:
+**  - All components >= 0 and < bi_radix.
+**  - Leading 0 components removed.
+**  - Sign either 1 or -1.
+**  - The number zero represented by a single 0 component and a sign of 1.
+*/
+static void
+normalize( real_bigint bi )
+    {
+    int c;
+
+    /* Borrow for negative components.  Got to be careful with the math here:
+    **   -9 / 10 == 0    -9 % 10 == -9
+    **   -10 / 10 == -1  -10 % 10 == 0
+    **   -11 / 10 == -1  -11 % 10 == -1
+    */
+    for ( c = 0; c < bi->num_comps - 1; ++c )
+	if ( bi->comps[c] < 0 )
+	    {
+	    bi->comps[c+1] += bi->comps[c] / bi_radix - 1;
+	    bi->comps[c] = bi->comps[c] % bi_radix;
+	    if ( bi->comps[c] != 0 )
+		bi->comps[c] += bi_radix;
+	    else
+		bi->comps[c+1] += 1;
+	    }
+    /* Is the top component negative? */
+    if ( bi->comps[bi->num_comps - 1] < 0 )
+	{
+	/* Switch the sign of the number, and fix up the components. */
+	bi->sign = -bi->sign;
+	for ( c = 0; c < bi->num_comps - 1; ++c )
+	    {
+	    bi->comps[c] =  bi_radix - bi->comps[c];
+	    bi->comps[c + 1] += 1;
+	    }
+	bi->comps[bi->num_comps - 1] = -bi->comps[bi->num_comps - 1];
+	}
+
+    /* Carry for components larger than the radix. */
+    for ( c = 0; c < bi->num_comps; ++c )
+	if ( bi->comps[c] >= bi_radix )
+	    {
+	    if ( c + 1 >= bi->num_comps )
+		more_comps( bi, bi->num_comps + 1 );
+	    bi->comps[c+1] += bi->comps[c] / bi_radix;
+	    bi->comps[c] = bi->comps[c] % bi_radix;
+	    }
+
+    /* Trim off any leading zero components. */
+    for ( ; bi->num_comps > 1 && bi->comps[bi->num_comps-1] == 0; --bi->num_comps )
+	;
+
+    /* Check for -0. */
+    if ( bi->num_comps == 1 && bi->comps[0] == 0 && bi->sign == -1 )
+	bi->sign = 1;
+    }
+
+
+static void
+check( real_bigint bi )
+    {
+    if ( check_level == 0 )
+	return;
+    if ( bi->refs == 0 )
+	{
+	(void) fprintf( stderr, "check: zero refs in bigint\n" );
+	(void) kill( getpid(), SIGFPE );
+	}
+    if ( bi->refs < 0 )
+	{
+	(void) fprintf( stderr, "check: negative refs in bigint\n" );
+	(void) kill( getpid(), SIGFPE );
+	}
+    if ( check_level < 3 )
+	{
+	/* At check levels less than 3, active bigints have a zero next. */
+	if ( bi->next != (real_bigint) 0 )
+	    {
+	    (void) fprintf(
+		stderr, "check: attempt to use a bigint from the free list\n" );
+	    (void) kill( getpid(), SIGFPE );
+	    }
+	}
+    else
+	{
+	/* At check levels 3 or higher, active bigints must be on the active
+	** list.
+	*/
+	real_bigint p;
+
+	for ( p = active_list; p != (real_bigint) 0; p = p->next )
+	    if ( p == bi )
+		break;
+	if ( p == (real_bigint) 0 )
+	    {
+	    (void) fprintf( stderr,
+		"check: attempt to use a bigint not on the active list\n" );
+	    (void) kill( getpid(), SIGFPE );
+	    }
+	}
+    if ( check_level >= 2 )
+	double_check();
+    if ( check_level >= 3 )
+	triple_check();
+    }
+
+
+static void
+double_check( void )
+    {
+    real_bigint p;
+    int c;
+
+    for ( p = free_list, c = 0; p != (real_bigint) 0; p = p->next, ++c )
+	if ( p->refs != 0 )
+	    {
+	    (void) fprintf( stderr,
+		"double_check: found a non-zero ref on the free list\n" );
+	    (void) kill( getpid(), SIGFPE );
+	    }
+    if ( c != free_count )
+	{
+	(void) fprintf( stderr,
+	    "double_check: free_count is %d but the free list has %d items\n",
+	    free_count, c );
+	(void) kill( getpid(), SIGFPE );
+	}
+    }
+
+
+static void
+triple_check( void )
+    {
+    real_bigint p;
+    int c;
+
+    for ( p = active_list, c = 0; p != (real_bigint) 0; p = p->next, ++c )
+	if ( p->refs == 0 )
+	    {
+	    (void) fprintf( stderr,
+		"triple_check: found a zero ref on the active list\n" );
+	    (void) kill( getpid(), SIGFPE );
+	    }
+    if ( c != active_count )
+	{
+	(void) fprintf( stderr,
+	    "triple_check: active_count is %d but active_list has %d items\n",
+	    free_count, c );
+	(void) kill( getpid(), SIGFPE );
+	}
+    }
+
+
+#ifdef DUMP
+/* Debug routine to dump out a complete bigint.  Does not consume bi. */
+static void
+dump( char* str, bigint obi )
+    {
+    int c;
+    real_bigint bi = (real_bigint) obi;
+
+    (void) fprintf( stdout, "dump %s at 0x%08x:\n", str, (unsigned int) bi );
+    (void) fprintf( stdout, "  refs: %d\n", bi->refs );
+    (void) fprintf( stdout, "  next: 0x%08x\n", (unsigned int) bi->next );
+    (void) fprintf( stdout, "  num_comps: %d\n", bi->num_comps );
+    (void) fprintf( stdout, "  max_comps: %d\n", bi->max_comps );
+    (void) fprintf( stdout, "  sign: %d\n", bi->sign );
+    for ( c = bi->num_comps - 1; c >= 0; --c )
+	(void) fprintf( stdout, "    comps[%d]: %11lld (0x%016llx)\n", c, (long long) bi->comps[c], (long long) bi->comps[c] );
+    (void) fprintf( stdout, "  print: " );
+    bi_print( stdout, bi_copy( bi ) );
+    (void) fprintf( stdout, "\n" );
+    }
+#endif /* DUMP */
+
+
+/* Trivial square-root routine so that we don't have to link in the math lib. */
+static int
+csqrt( comp c )
+    {
+    comp r, r2, diff;
+
+    if ( c < 0 )
+	{
+	(void) fprintf( stderr, "csqrt: imaginary result\n" );
+	(void) kill( getpid(), SIGFPE );
+	}
+
+    r = c / 2;
+    for (;;)
+	{
+	r2 = c / r;
+	diff = r - r2;
+	if ( diff == 0 || diff == -1 )
+	    return (int) r;
+	if ( diff == 1 )
+	    return (int) r2;
+	r = ( r + r2 ) / 2;
+	}
+    }
+
+
+/* Figure out how many bits are in a number. */
+static int
+cbits( comp c )
+    {
+    int b;
+
+    for ( b = 0; c != 0; ++b )
+	c >>= 1;
+    return b;
+    }
diff --git a/src/rt/bigint/low_primes.h b/src/rt/bigint/low_primes.h
new file mode 100644
index 00000000..c9d3df0b
--- /dev/null
+++ b/src/rt/bigint/low_primes.h
@@ -0,0 +1,1069 @@
+/* Primes up to 100000. */
+static long low_primes[] = {
+        2,     3,     5,     7,    11,    13,    17,    19,    23,
+       29,    31,    37,    41,    43,    47,    53,    59,    61,
+       67,    71,    73,    79,    83,    89,    97,   101,   103,
+      107,   109,   113,   127,   131,   137,   139,   149,   151,
+      157,   163,   167,   173,   179,   181,   191,   193,   197,
+      199,   211,   223,   227,   229,   233,   239,   241,   251,
+      257,   263,   269,   271,   277,   281,   283,   293,   307,
+      311,   313,   317,   331,   337,   347,   349,   353,   359,
+      367,   373,   379,   383,   389,   397,   401,   409,   419,
+      421,   431,   433,   439,   443,   449,   457,   461,   463,
+      467,   479,   487,   491,   499,   503,   509,   521,   523,
+      541,   547,   557,   563,   569,   571,   577,   587,   593,
+      599,   601,   607,   613,   617,   619,   631,   641,   643,
+      647,   653,   659,   661,   673,   677,   683,   691,   701,
+      709,   719,   727,   733,   739,   743,   751,   757,   761,
+      769,   773,   787,   797,   809,   811,   821,   823,   827,
+      829,   839,   853,   857,   859,   863,   877,   881,   883,
+      887,   907,   911,   919,   929,   937,   941,   947,   953,
+      967,   971,   977,   983,   991,   997,  1009,  1013,  1019,
+     1021,  1031,  1033,  1039,  1049,  1051,  1061,  1063,  1069,
+     1087,  1091,  1093,  1097,  1103,  1109,  1117,  1123,  1129,
+     1151,  1153,  1163,  1171,  1181,  1187,  1193,  1201,  1213,
+     1217,  1223,  1229,  1231,  1237,  1249,  1259,  1277,  1279,
+     1283,  1289,  1291,  1297,  1301,  1303,  1307,  1319,  1321,
+     1327,  1361,  1367,  1373,  1381,  1399,  1409,  1423,  1427,
+     1429,  1433,  1439,  1447,  1451,  1453,  1459,  1471,  1481,
+     1483,  1487,  1489,  1493,  1499,  1511,  1523,  1531,  1543,
+     1549,  1553,  1559,  1567,  1571,  1579,  1583,  1597,  1601,
+     1607,  1609,  1613,  1619,  1621,  1627,  1637,  1657,  1663,
+     1667,  1669,  1693,  1697,  1699,  1709,  1721,  1723,  1733,
+     1741,  1747,  1753,  1759,  1777,  1783,  1787,  1789,  1801,
+     1811,  1823,  1831,  1847,  1861,  1867,  1871,  1873,  1877,
+     1879,  1889,  1901,  1907,  1913,  1931,  1933,  1949,  1951,
+     1973,  1979,  1987,  1993,  1997,  1999,  2003,  2011,  2017,
+     2027,  2029,  2039,  2053,  2063,  2069,  2081,  2083,  2087,
+     2089,  2099,  2111,  2113,  2129,  2131,  2137,  2141,  2143,
+     2153,  2161,  2179,  2203,  2207,  2213,  2221,  2237,  2239,
+     2243,  2251,  2267,  2269,  2273,  2281,  2287,  2293,  2297,
+     2309,  2311,  2333,  2339,  2341,  2347,  2351,  2357,  2371,
+     2377,  2381,  2383,  2389,  2393,  2399,  2411,  2417,  2423,
+     2437,  2441,  2447,  2459,  2467,  2473,  2477,  2503,  2521,
+     2531,  2539,  2543,  2549,  2551,  2557,  2579,  2591,  2593,
+     2609,  2617,  2621,  2633,  2647,  2657,  2659,  2663,  2671,
+     2677,  2683,  2687,  2689,  2693,  2699,  2707,  2711,  2713,
+     2719,  2729,  2731,  2741,  2749,  2753,  2767,  2777,  2789,
+     2791,  2797,  2801,  2803,  2819,  2833,  2837,  2843,  2851,
+     2857,  2861,  2879,  2887,  2897,  2903,  2909,  2917,  2927,
+     2939,  2953,  2957,  2963,  2969,  2971,  2999,  3001,  3011,
+     3019,  3023,  3037,  3041,  3049,  3061,  3067,  3079,  3083,
+     3089,  3109,  3119,  3121,  3137,  3163,  3167,  3169,  3181,
+     3187,  3191,  3203,  3209,  3217,  3221,  3229,  3251,  3253,
+     3257,  3259,  3271,  3299,  3301,  3307,  3313,  3319,  3323,
+     3329,  3331,  3343,  3347,  3359,  3361,  3371,  3373,  3389,
+     3391,  3407,  3413,  3433,  3449,  3457,  3461,  3463,  3467,
+     3469,  3491,  3499,  3511,  3517,  3527,  3529,  3533,  3539,
+     3541,  3547,  3557,  3559,  3571,  3581,  3583,  3593,  3607,
+     3613,  3617,  3623,  3631,  3637,  3643,  3659,  3671,  3673,
+     3677,  3691,  3697,  3701,  3709,  3719,  3727,  3733,  3739,
+     3761,  3767,  3769,  3779,  3793,  3797,  3803,  3821,  3823,
+     3833,  3847,  3851,  3853,  3863,  3877,  3881,  3889,  3907,
+     3911,  3917,  3919,  3923,  3929,  3931,  3943,  3947,  3967,
+     3989,  4001,  4003,  4007,  4013,  4019,  4021,  4027,  4049,
+     4051,  4057,  4073,  4079,  4091,  4093,  4099,  4111,  4127,
+     4129,  4133,  4139,  4153,  4157,  4159,  4177,  4201,  4211,
+     4217,  4219,  4229,  4231,  4241,  4243,  4253,  4259,  4261,
+     4271,  4273,  4283,  4289,  4297,  4327,  4337,  4339,  4349,
+     4357,  4363,  4373,  4391,  4397,  4409,  4421,  4423,  4441,
+     4447,  4451,  4457,  4463,  4481,  4483,  4493,  4507,  4513,
+     4517,  4519,  4523,  4547,  4549,  4561,  4567,  4583,  4591,
+     4597,  4603,  4621,  4637,  4639,  4643,  4649,  4651,  4657,
+     4663,  4673,  4679,  4691,  4703,  4721,  4723,  4729,  4733,
+     4751,  4759,  4783,  4787,  4789,  4793,  4799,  4801,  4813,
+     4817,  4831,  4861,  4871,  4877,  4889,  4903,  4909,  4919,
+     4931,  4933,  4937,  4943,  4951,  4957,  4967,  4969,  4973,
+     4987,  4993,  4999,  5003,  5009,  5011,  5021,  5023,  5039,
+     5051,  5059,  5077,  5081,  5087,  5099,  5101,  5107,  5113,
+     5119,  5147,  5153,  5167,  5171,  5179,  5189,  5197,  5209,
+     5227,  5231,  5233,  5237,  5261,  5273,  5279,  5281,  5297,
+     5303,  5309,  5323,  5333,  5347,  5351,  5381,  5387,  5393,
+     5399,  5407,  5413,  5417,  5419,  5431,  5437,  5441,  5443,
+     5449,  5471,  5477,  5479,  5483,  5501,  5503,  5507,  5519,
+     5521,  5527,  5531,  5557,  5563,  5569,  5573,  5581,  5591,
+     5623,  5639,  5641,  5647,  5651,  5653,  5657,  5659,  5669,
+     5683,  5689,  5693,  5701,  5711,  5717,  5737,  5741,  5743,
+     5749,  5779,  5783,  5791,  5801,  5807,  5813,  5821,  5827,
+     5839,  5843,  5849,  5851,  5857,  5861,  5867,  5869,  5879,
+     5881,  5897,  5903,  5923,  5927,  5939,  5953,  5981,  5987,
+     6007,  6011,  6029,  6037,  6043,  6047,  6053,  6067,  6073,
+     6079,  6089,  6091,  6101,  6113,  6121,  6131,  6133,  6143,
+     6151,  6163,  6173,  6197,  6199,  6203,  6211,  6217,  6221,
+     6229,  6247,  6257,  6263,  6269,  6271,  6277,  6287,  6299,
+     6301,  6311,  6317,  6323,  6329,  6337,  6343,  6353,  6359,
+     6361,  6367,  6373,  6379,  6389,  6397,  6421,  6427,  6449,
+     6451,  6469,  6473,  6481,  6491,  6521,  6529,  6547,  6551,
+     6553,  6563,  6569,  6571,  6577,  6581,  6599,  6607,  6619,
+     6637,  6653,  6659,  6661,  6673,  6679,  6689,  6691,  6701,
+     6703,  6709,  6719,  6733,  6737,  6761,  6763,  6779,  6781,
+     6791,  6793,  6803,  6823,  6827,  6829,  6833,  6841,  6857,
+     6863,  6869,  6871,  6883,  6899,  6907,  6911,  6917,  6947,
+     6949,  6959,  6961,  6967,  6971,  6977,  6983,  6991,  6997,
+     7001,  7013,  7019,  7027,  7039,  7043,  7057,  7069,  7079,
+     7103,  7109,  7121,  7127,  7129,  7151,  7159,  7177,  7187,
+     7193,  7207,  7211,  7213,  7219,  7229,  7237,  7243,  7247,
+     7253,  7283,  7297,  7307,  7309,  7321,  7331,  7333,  7349,
+     7351,  7369,  7393,  7411,  7417,  7433,  7451,  7457,  7459,
+     7477,  7481,  7487,  7489,  7499,  7507,  7517,  7523,  7529,
+     7537,  7541,  7547,  7549,  7559,  7561,  7573,  7577,  7583,
+     7589,  7591,  7603,  7607,  7621,  7639,  7643,  7649,  7669,
+     7673,  7681,  7687,  7691,  7699,  7703,  7717,  7723,  7727,
+     7741,  7753,  7757,  7759,  7789,  7793,  7817,  7823,  7829,
+     7841,  7853,  7867,  7873,  7877,  7879,  7883,  7901,  7907,
+     7919,  7927,  7933,  7937,  7949,  7951,  7963,  7993,  8009,
+     8011,  8017,  8039,  8053,  8059,  8069,  8081,  8087,  8089,
+     8093,  8101,  8111,  8117,  8123,  8147,  8161,  8167,  8171,
+     8179,  8191,  8209,  8219,  8221,  8231,  8233,  8237,  8243,
+     8263,  8269,  8273,  8287,  8291,  8293,  8297,  8311,  8317,
+     8329,  8353,  8363,  8369,  8377,  8387,  8389,  8419,  8423,
+     8429,  8431,  8443,  8447,  8461,  8467,  8501,  8513,  8521,
+     8527,  8537,  8539,  8543,  8563,  8573,  8581,  8597,  8599,
+     8609,  8623,  8627,  8629,  8641,  8647,  8663,  8669,  8677,
+     8681,  8689,  8693,  8699,  8707,  8713,  8719,  8731,  8737,
+     8741,  8747,  8753,  8761,  8779,  8783,  8803,  8807,  8819,
+     8821,  8831,  8837,  8839,  8849,  8861,  8863,  8867,  8887,
+     8893,  8923,  8929,  8933,  8941,  8951,  8963,  8969,  8971,
+     8999,  9001,  9007,  9011,  9013,  9029,  9041,  9043,  9049,
+     9059,  9067,  9091,  9103,  9109,  9127,  9133,  9137,  9151,
+     9157,  9161,  9173,  9181,  9187,  9199,  9203,  9209,  9221,
+     9227,  9239,  9241,  9257,  9277,  9281,  9283,  9293,  9311,
+     9319,  9323,  9337,  9341,  9343,  9349,  9371,  9377,  9391,
+     9397,  9403,  9413,  9419,  9421,  9431,  9433,  9437,  9439,
+     9461,  9463,  9467,  9473,  9479,  9491,  9497,  9511,  9521,
+     9533,  9539,  9547,  9551,  9587,  9601,  9613,  9619,  9623,
+     9629,  9631,  9643,  9649,  9661,  9677,  9679,  9689,  9697,
+     9719,  9721,  9733,  9739,  9743,  9749,  9767,  9769,  9781,
+     9787,  9791,  9803,  9811,  9817,  9829,  9833,  9839,  9851,
+     9857,  9859,  9871,  9883,  9887,  9901,  9907,  9923,  9929,
+     9931,  9941,  9949,  9967,  9973, 10007, 10009, 10037, 10039,
+    10061, 10067, 10069, 10079, 10091, 10093, 10099, 10103, 10111,
+    10133, 10139, 10141, 10151, 10159, 10163, 10169, 10177, 10181,
+    10193, 10211, 10223, 10243, 10247, 10253, 10259, 10267, 10271,
+    10273, 10289, 10301, 10303, 10313, 10321, 10331, 10333, 10337,
+    10343, 10357, 10369, 10391, 10399, 10427, 10429, 10433, 10453,
+    10457, 10459, 10463, 10477, 10487, 10499, 10501, 10513, 10529,
+    10531, 10559, 10567, 10589, 10597, 10601, 10607, 10613, 10627,
+    10631, 10639, 10651, 10657, 10663, 10667, 10687, 10691, 10709,
+    10711, 10723, 10729, 10733, 10739, 10753, 10771, 10781, 10789,
+    10799, 10831, 10837, 10847, 10853, 10859, 10861, 10867, 10883,
+    10889, 10891, 10903, 10909, 10937, 10939, 10949, 10957, 10973,
+    10979, 10987, 10993, 11003, 11027, 11047, 11057, 11059, 11069,
+    11071, 11083, 11087, 11093, 11113, 11117, 11119, 11131, 11149,
+    11159, 11161, 11171, 11173, 11177, 11197, 11213, 11239, 11243,
+    11251, 11257, 11261, 11273, 11279, 11287, 11299, 11311, 11317,
+    11321, 11329, 11351, 11353, 11369, 11383, 11393, 11399, 11411,
+    11423, 11437, 11443, 11447, 11467, 11471, 11483, 11489, 11491,
+    11497, 11503, 11519, 11527, 11549, 11551, 11579, 11587, 11593,
+    11597, 11617, 11621, 11633, 11657, 11677, 11681, 11689, 11699,
+    11701, 11717, 11719, 11731, 11743, 11777, 11779, 11783, 11789,
+    11801, 11807, 11813, 11821, 11827, 11831, 11833, 11839, 11863,
+    11867, 11887, 11897, 11903, 11909, 11923, 11927, 11933, 11939,
+    11941, 11953, 11959, 11969, 11971, 11981, 11987, 12007, 12011,
+    12037, 12041, 12043, 12049, 12071, 12073, 12097, 12101, 12107,
+    12109, 12113, 12119, 12143, 12149, 12157, 12161, 12163, 12197,
+    12203, 12211, 12227, 12239, 12241, 12251, 12253, 12263, 12269,
+    12277, 12281, 12289, 12301, 12323, 12329, 12343, 12347, 12373,
+    12377, 12379, 12391, 12401, 12409, 12413, 12421, 12433, 12437,
+    12451, 12457, 12473, 12479, 12487, 12491, 12497, 12503, 12511,
+    12517, 12527, 12539, 12541, 12547, 12553, 12569, 12577, 12583,
+    12589, 12601, 12611, 12613, 12619, 12637, 12641, 12647, 12653,
+    12659, 12671, 12689, 12697, 12703, 12713, 12721, 12739, 12743,
+    12757, 12763, 12781, 12791, 12799, 12809, 12821, 12823, 12829,
+    12841, 12853, 12889, 12893, 12899, 12907, 12911, 12917, 12919,
+    12923, 12941, 12953, 12959, 12967, 12973, 12979, 12983, 13001,
+    13003, 13007, 13009, 13033, 13037, 13043, 13049, 13063, 13093,
+    13099, 13103, 13109, 13121, 13127, 13147, 13151, 13159, 13163,
+    13171, 13177, 13183, 13187, 13217, 13219, 13229, 13241, 13249,
+    13259, 13267, 13291, 13297, 13309, 13313, 13327, 13331, 13337,
+    13339, 13367, 13381, 13397, 13399, 13411, 13417, 13421, 13441,
+    13451, 13457, 13463, 13469, 13477, 13487, 13499, 13513, 13523,
+    13537, 13553, 13567, 13577, 13591, 13597, 13613, 13619, 13627,
+    13633, 13649, 13669, 13679, 13681, 13687, 13691, 13693, 13697,
+    13709, 13711, 13721, 13723, 13729, 13751, 13757, 13759, 13763,
+    13781, 13789, 13799, 13807, 13829, 13831, 13841, 13859, 13873,
+    13877, 13879, 13883, 13901, 13903, 13907, 13913, 13921, 13931,
+    13933, 13963, 13967, 13997, 13999, 14009, 14011, 14029, 14033,
+    14051, 14057, 14071, 14081, 14083, 14087, 14107, 14143, 14149,
+    14153, 14159, 14173, 14177, 14197, 14207, 14221, 14243, 14249,
+    14251, 14281, 14293, 14303, 14321, 14323, 14327, 14341, 14347,
+    14369, 14387, 14389, 14401, 14407, 14411, 14419, 14423, 14431,
+    14437, 14447, 14449, 14461, 14479, 14489, 14503, 14519, 14533,
+    14537, 14543, 14549, 14551, 14557, 14561, 14563, 14591, 14593,
+    14621, 14627, 14629, 14633, 14639, 14653, 14657, 14669, 14683,
+    14699, 14713, 14717, 14723, 14731, 14737, 14741, 14747, 14753,
+    14759, 14767, 14771, 14779, 14783, 14797, 14813, 14821, 14827,
+    14831, 14843, 14851, 14867, 14869, 14879, 14887, 14891, 14897,
+    14923, 14929, 14939, 14947, 14951, 14957, 14969, 14983, 15013,
+    15017, 15031, 15053, 15061, 15073, 15077, 15083, 15091, 15101,
+    15107, 15121, 15131, 15137, 15139, 15149, 15161, 15173, 15187,
+    15193, 15199, 15217, 15227, 15233, 15241, 15259, 15263, 15269,
+    15271, 15277, 15287, 15289, 15299, 15307, 15313, 15319, 15329,
+    15331, 15349, 15359, 15361, 15373, 15377, 15383, 15391, 15401,
+    15413, 15427, 15439, 15443, 15451, 15461, 15467, 15473, 15493,
+    15497, 15511, 15527, 15541, 15551, 15559, 15569, 15581, 15583,
+    15601, 15607, 15619, 15629, 15641, 15643, 15647, 15649, 15661,
+    15667, 15671, 15679, 15683, 15727, 15731, 15733, 15737, 15739,
+    15749, 15761, 15767, 15773, 15787, 15791, 15797, 15803, 15809,
+    15817, 15823, 15859, 15877, 15881, 15887, 15889, 15901, 15907,
+    15913, 15919, 15923, 15937, 15959, 15971, 15973, 15991, 16001,
+    16007, 16033, 16057, 16061, 16063, 16067, 16069, 16073, 16087,
+    16091, 16097, 16103, 16111, 16127, 16139, 16141, 16183, 16187,
+    16189, 16193, 16217, 16223, 16229, 16231, 16249, 16253, 16267,
+    16273, 16301, 16319, 16333, 16339, 16349, 16361, 16363, 16369,
+    16381, 16411, 16417, 16421, 16427, 16433, 16447, 16451, 16453,
+    16477, 16481, 16487, 16493, 16519, 16529, 16547, 16553, 16561,
+    16567, 16573, 16603, 16607, 16619, 16631, 16633, 16649, 16651,
+    16657, 16661, 16673, 16691, 16693, 16699, 16703, 16729, 16741,
+    16747, 16759, 16763, 16787, 16811, 16823, 16829, 16831, 16843,
+    16871, 16879, 16883, 16889, 16901, 16903, 16921, 16927, 16931,
+    16937, 16943, 16963, 16979, 16981, 16987, 16993, 17011, 17021,
+    17027, 17029, 17033, 17041, 17047, 17053, 17077, 17093, 17099,
+    17107, 17117, 17123, 17137, 17159, 17167, 17183, 17189, 17191,
+    17203, 17207, 17209, 17231, 17239, 17257, 17291, 17293, 17299,
+    17317, 17321, 17327, 17333, 17341, 17351, 17359, 17377, 17383,
+    17387, 17389, 17393, 17401, 17417, 17419, 17431, 17443, 17449,
+    17467, 17471, 17477, 17483, 17489, 17491, 17497, 17509, 17519,
+    17539, 17551, 17569, 17573, 17579, 17581, 17597, 17599, 17609,
+    17623, 17627, 17657, 17659, 17669, 17681, 17683, 17707, 17713,
+    17729, 17737, 17747, 17749, 17761, 17783, 17789, 17791, 17807,
+    17827, 17837, 17839, 17851, 17863, 17881, 17891, 17903, 17909,
+    17911, 17921, 17923, 17929, 17939, 17957, 17959, 17971, 17977,
+    17981, 17987, 17989, 18013, 18041, 18043, 18047, 18049, 18059,
+    18061, 18077, 18089, 18097, 18119, 18121, 18127, 18131, 18133,
+    18143, 18149, 18169, 18181, 18191, 18199, 18211, 18217, 18223,
+    18229, 18233, 18251, 18253, 18257, 18269, 18287, 18289, 18301,
+    18307, 18311, 18313, 18329, 18341, 18353, 18367, 18371, 18379,
+    18397, 18401, 18413, 18427, 18433, 18439, 18443, 18451, 18457,
+    18461, 18481, 18493, 18503, 18517, 18521, 18523, 18539, 18541,
+    18553, 18583, 18587, 18593, 18617, 18637, 18661, 18671, 18679,
+    18691, 18701, 18713, 18719, 18731, 18743, 18749, 18757, 18773,
+    18787, 18793, 18797, 18803, 18839, 18859, 18869, 18899, 18911,
+    18913, 18917, 18919, 18947, 18959, 18973, 18979, 19001, 19009,
+    19013, 19031, 19037, 19051, 19069, 19073, 19079, 19081, 19087,
+    19121, 19139, 19141, 19157, 19163, 19181, 19183, 19207, 19211,
+    19213, 19219, 19231, 19237, 19249, 19259, 19267, 19273, 19289,
+    19301, 19309, 19319, 19333, 19373, 19379, 19381, 19387, 19391,
+    19403, 19417, 19421, 19423, 19427, 19429, 19433, 19441, 19447,
+    19457, 19463, 19469, 19471, 19477, 19483, 19489, 19501, 19507,
+    19531, 19541, 19543, 19553, 19559, 19571, 19577, 19583, 19597,
+    19603, 19609, 19661, 19681, 19687, 19697, 19699, 19709, 19717,
+    19727, 19739, 19751, 19753, 19759, 19763, 19777, 19793, 19801,
+    19813, 19819, 19841, 19843, 19853, 19861, 19867, 19889, 19891,
+    19913, 19919, 19927, 19937, 19949, 19961, 19963, 19973, 19979,
+    19991, 19993, 19997, 20011, 20021, 20023, 20029, 20047, 20051,
+    20063, 20071, 20089, 20101, 20107, 20113, 20117, 20123, 20129,
+    20143, 20147, 20149, 20161, 20173, 20177, 20183, 20201, 20219,
+    20231, 20233, 20249, 20261, 20269, 20287, 20297, 20323, 20327,
+    20333, 20341, 20347, 20353, 20357, 20359, 20369, 20389, 20393,
+    20399, 20407, 20411, 20431, 20441, 20443, 20477, 20479, 20483,
+    20507, 20509, 20521, 20533, 20543, 20549, 20551, 20563, 20593,
+    20599, 20611, 20627, 20639, 20641, 20663, 20681, 20693, 20707,
+    20717, 20719, 20731, 20743, 20747, 20749, 20753, 20759, 20771,
+    20773, 20789, 20807, 20809, 20849, 20857, 20873, 20879, 20887,
+    20897, 20899, 20903, 20921, 20929, 20939, 20947, 20959, 20963,
+    20981, 20983, 21001, 21011, 21013, 21017, 21019, 21023, 21031,
+    21059, 21061, 21067, 21089, 21101, 21107, 21121, 21139, 21143,
+    21149, 21157, 21163, 21169, 21179, 21187, 21191, 21193, 21211,
+    21221, 21227, 21247, 21269, 21277, 21283, 21313, 21317, 21319,
+    21323, 21341, 21347, 21377, 21379, 21383, 21391, 21397, 21401,
+    21407, 21419, 21433, 21467, 21481, 21487, 21491, 21493, 21499,
+    21503, 21517, 21521, 21523, 21529, 21557, 21559, 21563, 21569,
+    21577, 21587, 21589, 21599, 21601, 21611, 21613, 21617, 21647,
+    21649, 21661, 21673, 21683, 21701, 21713, 21727, 21737, 21739,
+    21751, 21757, 21767, 21773, 21787, 21799, 21803, 21817, 21821,
+    21839, 21841, 21851, 21859, 21863, 21871, 21881, 21893, 21911,
+    21929, 21937, 21943, 21961, 21977, 21991, 21997, 22003, 22013,
+    22027, 22031, 22037, 22039, 22051, 22063, 22067, 22073, 22079,
+    22091, 22093, 22109, 22111, 22123, 22129, 22133, 22147, 22153,
+    22157, 22159, 22171, 22189, 22193, 22229, 22247, 22259, 22271,
+    22273, 22277, 22279, 22283, 22291, 22303, 22307, 22343, 22349,
+    22367, 22369, 22381, 22391, 22397, 22409, 22433, 22441, 22447,
+    22453, 22469, 22481, 22483, 22501, 22511, 22531, 22541, 22543,
+    22549, 22567, 22571, 22573, 22613, 22619, 22621, 22637, 22639,
+    22643, 22651, 22669, 22679, 22691, 22697, 22699, 22709, 22717,
+    22721, 22727, 22739, 22741, 22751, 22769, 22777, 22783, 22787,
+    22807, 22811, 22817, 22853, 22859, 22861, 22871, 22877, 22901,
+    22907, 22921, 22937, 22943, 22961, 22963, 22973, 22993, 23003,
+    23011, 23017, 23021, 23027, 23029, 23039, 23041, 23053, 23057,
+    23059, 23063, 23071, 23081, 23087, 23099, 23117, 23131, 23143,
+    23159, 23167, 23173, 23189, 23197, 23201, 23203, 23209, 23227,
+    23251, 23269, 23279, 23291, 23293, 23297, 23311, 23321, 23327,
+    23333, 23339, 23357, 23369, 23371, 23399, 23417, 23431, 23447,
+    23459, 23473, 23497, 23509, 23531, 23537, 23539, 23549, 23557,
+    23561, 23563, 23567, 23581, 23593, 23599, 23603, 23609, 23623,
+    23627, 23629, 23633, 23663, 23669, 23671, 23677, 23687, 23689,
+    23719, 23741, 23743, 23747, 23753, 23761, 23767, 23773, 23789,
+    23801, 23813, 23819, 23827, 23831, 23833, 23857, 23869, 23873,
+    23879, 23887, 23893, 23899, 23909, 23911, 23917, 23929, 23957,
+    23971, 23977, 23981, 23993, 24001, 24007, 24019, 24023, 24029,
+    24043, 24049, 24061, 24071, 24077, 24083, 24091, 24097, 24103,
+    24107, 24109, 24113, 24121, 24133, 24137, 24151, 24169, 24179,
+    24181, 24197, 24203, 24223, 24229, 24239, 24247, 24251, 24281,
+    24317, 24329, 24337, 24359, 24371, 24373, 24379, 24391, 24407,
+    24413, 24419, 24421, 24439, 24443, 24469, 24473, 24481, 24499,
+    24509, 24517, 24527, 24533, 24547, 24551, 24571, 24593, 24611,
+    24623, 24631, 24659, 24671, 24677, 24683, 24691, 24697, 24709,
+    24733, 24749, 24763, 24767, 24781, 24793, 24799, 24809, 24821,
+    24841, 24847, 24851, 24859, 24877, 24889, 24907, 24917, 24919,
+    24923, 24943, 24953, 24967, 24971, 24977, 24979, 24989, 25013,
+    25031, 25033, 25037, 25057, 25073, 25087, 25097, 25111, 25117,
+    25121, 25127, 25147, 25153, 25163, 25169, 25171, 25183, 25189,
+    25219, 25229, 25237, 25243, 25247, 25253, 25261, 25301, 25303,
+    25307, 25309, 25321, 25339, 25343, 25349, 25357, 25367, 25373,
+    25391, 25409, 25411, 25423, 25439, 25447, 25453, 25457, 25463,
+    25469, 25471, 25523, 25537, 25541, 25561, 25577, 25579, 25583,
+    25589, 25601, 25603, 25609, 25621, 25633, 25639, 25643, 25657,
+    25667, 25673, 25679, 25693, 25703, 25717, 25733, 25741, 25747,
+    25759, 25763, 25771, 25793, 25799, 25801, 25819, 25841, 25847,
+    25849, 25867, 25873, 25889, 25903, 25913, 25919, 25931, 25933,
+    25939, 25943, 25951, 25969, 25981, 25997, 25999, 26003, 26017,
+    26021, 26029, 26041, 26053, 26083, 26099, 26107, 26111, 26113,
+    26119, 26141, 26153, 26161, 26171, 26177, 26183, 26189, 26203,
+    26209, 26227, 26237, 26249, 26251, 26261, 26263, 26267, 26293,
+    26297, 26309, 26317, 26321, 26339, 26347, 26357, 26371, 26387,
+    26393, 26399, 26407, 26417, 26423, 26431, 26437, 26449, 26459,
+    26479, 26489, 26497, 26501, 26513, 26539, 26557, 26561, 26573,
+    26591, 26597, 26627, 26633, 26641, 26647, 26669, 26681, 26683,
+    26687, 26693, 26699, 26701, 26711, 26713, 26717, 26723, 26729,
+    26731, 26737, 26759, 26777, 26783, 26801, 26813, 26821, 26833,
+    26839, 26849, 26861, 26863, 26879, 26881, 26891, 26893, 26903,
+    26921, 26927, 26947, 26951, 26953, 26959, 26981, 26987, 26993,
+    27011, 27017, 27031, 27043, 27059, 27061, 27067, 27073, 27077,
+    27091, 27103, 27107, 27109, 27127, 27143, 27179, 27191, 27197,
+    27211, 27239, 27241, 27253, 27259, 27271, 27277, 27281, 27283,
+    27299, 27329, 27337, 27361, 27367, 27397, 27407, 27409, 27427,
+    27431, 27437, 27449, 27457, 27479, 27481, 27487, 27509, 27527,
+    27529, 27539, 27541, 27551, 27581, 27583, 27611, 27617, 27631,
+    27647, 27653, 27673, 27689, 27691, 27697, 27701, 27733, 27737,
+    27739, 27743, 27749, 27751, 27763, 27767, 27773, 27779, 27791,
+    27793, 27799, 27803, 27809, 27817, 27823, 27827, 27847, 27851,
+    27883, 27893, 27901, 27917, 27919, 27941, 27943, 27947, 27953,
+    27961, 27967, 27983, 27997, 28001, 28019, 28027, 28031, 28051,
+    28057, 28069, 28081, 28087, 28097, 28099, 28109, 28111, 28123,
+    28151, 28163, 28181, 28183, 28201, 28211, 28219, 28229, 28277,
+    28279, 28283, 28289, 28297, 28307, 28309, 28319, 28349, 28351,
+    28387, 28393, 28403, 28409, 28411, 28429, 28433, 28439, 28447,
+    28463, 28477, 28493, 28499, 28513, 28517, 28537, 28541, 28547,
+    28549, 28559, 28571, 28573, 28579, 28591, 28597, 28603, 28607,
+    28619, 28621, 28627, 28631, 28643, 28649, 28657, 28661, 28663,
+    28669, 28687, 28697, 28703, 28711, 28723, 28729, 28751, 28753,
+    28759, 28771, 28789, 28793, 28807, 28813, 28817, 28837, 28843,
+    28859, 28867, 28871, 28879, 28901, 28909, 28921, 28927, 28933,
+    28949, 28961, 28979, 29009, 29017, 29021, 29023, 29027, 29033,
+    29059, 29063, 29077, 29101, 29123, 29129, 29131, 29137, 29147,
+    29153, 29167, 29173, 29179, 29191, 29201, 29207, 29209, 29221,
+    29231, 29243, 29251, 29269, 29287, 29297, 29303, 29311, 29327,
+    29333, 29339, 29347, 29363, 29383, 29387, 29389, 29399, 29401,
+    29411, 29423, 29429, 29437, 29443, 29453, 29473, 29483, 29501,
+    29527, 29531, 29537, 29567, 29569, 29573, 29581, 29587, 29599,
+    29611, 29629, 29633, 29641, 29663, 29669, 29671, 29683, 29717,
+    29723, 29741, 29753, 29759, 29761, 29789, 29803, 29819, 29833,
+    29837, 29851, 29863, 29867, 29873, 29879, 29881, 29917, 29921,
+    29927, 29947, 29959, 29983, 29989, 30011, 30013, 30029, 30047,
+    30059, 30071, 30089, 30091, 30097, 30103, 30109, 30113, 30119,
+    30133, 30137, 30139, 30161, 30169, 30181, 30187, 30197, 30203,
+    30211, 30223, 30241, 30253, 30259, 30269, 30271, 30293, 30307,
+    30313, 30319, 30323, 30341, 30347, 30367, 30389, 30391, 30403,
+    30427, 30431, 30449, 30467, 30469, 30491, 30493, 30497, 30509,
+    30517, 30529, 30539, 30553, 30557, 30559, 30577, 30593, 30631,
+    30637, 30643, 30649, 30661, 30671, 30677, 30689, 30697, 30703,
+    30707, 30713, 30727, 30757, 30763, 30773, 30781, 30803, 30809,
+    30817, 30829, 30839, 30841, 30851, 30853, 30859, 30869, 30871,
+    30881, 30893, 30911, 30931, 30937, 30941, 30949, 30971, 30977,
+    30983, 31013, 31019, 31033, 31039, 31051, 31063, 31069, 31079,
+    31081, 31091, 31121, 31123, 31139, 31147, 31151, 31153, 31159,
+    31177, 31181, 31183, 31189, 31193, 31219, 31223, 31231, 31237,
+    31247, 31249, 31253, 31259, 31267, 31271, 31277, 31307, 31319,
+    31321, 31327, 31333, 31337, 31357, 31379, 31387, 31391, 31393,
+    31397, 31469, 31477, 31481, 31489, 31511, 31513, 31517, 31531,
+    31541, 31543, 31547, 31567, 31573, 31583, 31601, 31607, 31627,
+    31643, 31649, 31657, 31663, 31667, 31687, 31699, 31721, 31723,
+    31727, 31729, 31741, 31751, 31769, 31771, 31793, 31799, 31817,
+    31847, 31849, 31859, 31873, 31883, 31891, 31907, 31957, 31963,
+    31973, 31981, 31991, 32003, 32009, 32027, 32029, 32051, 32057,
+    32059, 32063, 32069, 32077, 32083, 32089, 32099, 32117, 32119,
+    32141, 32143, 32159, 32173, 32183, 32189, 32191, 32203, 32213,
+    32233, 32237, 32251, 32257, 32261, 32297, 32299, 32303, 32309,
+    32321, 32323, 32327, 32341, 32353, 32359, 32363, 32369, 32371,
+    32377, 32381, 32401, 32411, 32413, 32423, 32429, 32441, 32443,
+    32467, 32479, 32491, 32497, 32503, 32507, 32531, 32533, 32537,
+    32561, 32563, 32569, 32573, 32579, 32587, 32603, 32609, 32611,
+    32621, 32633, 32647, 32653, 32687, 32693, 32707, 32713, 32717,
+    32719, 32749, 32771, 32779, 32783, 32789, 32797, 32801, 32803,
+    32831, 32833, 32839, 32843, 32869, 32887, 32909, 32911, 32917,
+    32933, 32939, 32941, 32957, 32969, 32971, 32983, 32987, 32993,
+    32999, 33013, 33023, 33029, 33037, 33049, 33053, 33071, 33073,
+    33083, 33091, 33107, 33113, 33119, 33149, 33151, 33161, 33179,
+    33181, 33191, 33199, 33203, 33211, 33223, 33247, 33287, 33289,
+    33301, 33311, 33317, 33329, 33331, 33343, 33347, 33349, 33353,
+    33359, 33377, 33391, 33403, 33409, 33413, 33427, 33457, 33461,
+    33469, 33479, 33487, 33493, 33503, 33521, 33529, 33533, 33547,
+    33563, 33569, 33577, 33581, 33587, 33589, 33599, 33601, 33613,
+    33617, 33619, 33623, 33629, 33637, 33641, 33647, 33679, 33703,
+    33713, 33721, 33739, 33749, 33751, 33757, 33767, 33769, 33773,
+    33791, 33797, 33809, 33811, 33827, 33829, 33851, 33857, 33863,
+    33871, 33889, 33893, 33911, 33923, 33931, 33937, 33941, 33961,
+    33967, 33997, 34019, 34031, 34033, 34039, 34057, 34061, 34123,
+    34127, 34129, 34141, 34147, 34157, 34159, 34171, 34183, 34211,
+    34213, 34217, 34231, 34253, 34259, 34261, 34267, 34273, 34283,
+    34297, 34301, 34303, 34313, 34319, 34327, 34337, 34351, 34361,
+    34367, 34369, 34381, 34403, 34421, 34429, 34439, 34457, 34469,
+    34471, 34483, 34487, 34499, 34501, 34511, 34513, 34519, 34537,
+    34543, 34549, 34583, 34589, 34591, 34603, 34607, 34613, 34631,
+    34649, 34651, 34667, 34673, 34679, 34687, 34693, 34703, 34721,
+    34729, 34739, 34747, 34757, 34759, 34763, 34781, 34807, 34819,
+    34841, 34843, 34847, 34849, 34871, 34877, 34883, 34897, 34913,
+    34919, 34939, 34949, 34961, 34963, 34981, 35023, 35027, 35051,
+    35053, 35059, 35069, 35081, 35083, 35089, 35099, 35107, 35111,
+    35117, 35129, 35141, 35149, 35153, 35159, 35171, 35201, 35221,
+    35227, 35251, 35257, 35267, 35279, 35281, 35291, 35311, 35317,
+    35323, 35327, 35339, 35353, 35363, 35381, 35393, 35401, 35407,
+    35419, 35423, 35437, 35447, 35449, 35461, 35491, 35507, 35509,
+    35521, 35527, 35531, 35533, 35537, 35543, 35569, 35573, 35591,
+    35593, 35597, 35603, 35617, 35671, 35677, 35729, 35731, 35747,
+    35753, 35759, 35771, 35797, 35801, 35803, 35809, 35831, 35837,
+    35839, 35851, 35863, 35869, 35879, 35897, 35899, 35911, 35923,
+    35933, 35951, 35963, 35969, 35977, 35983, 35993, 35999, 36007,
+    36011, 36013, 36017, 36037, 36061, 36067, 36073, 36083, 36097,
+    36107, 36109, 36131, 36137, 36151, 36161, 36187, 36191, 36209,
+    36217, 36229, 36241, 36251, 36263, 36269, 36277, 36293, 36299,
+    36307, 36313, 36319, 36341, 36343, 36353, 36373, 36383, 36389,
+    36433, 36451, 36457, 36467, 36469, 36473, 36479, 36493, 36497,
+    36523, 36527, 36529, 36541, 36551, 36559, 36563, 36571, 36583,
+    36587, 36599, 36607, 36629, 36637, 36643, 36653, 36671, 36677,
+    36683, 36691, 36697, 36709, 36713, 36721, 36739, 36749, 36761,
+    36767, 36779, 36781, 36787, 36791, 36793, 36809, 36821, 36833,
+    36847, 36857, 36871, 36877, 36887, 36899, 36901, 36913, 36919,
+    36923, 36929, 36931, 36943, 36947, 36973, 36979, 36997, 37003,
+    37013, 37019, 37021, 37039, 37049, 37057, 37061, 37087, 37097,
+    37117, 37123, 37139, 37159, 37171, 37181, 37189, 37199, 37201,
+    37217, 37223, 37243, 37253, 37273, 37277, 37307, 37309, 37313,
+    37321, 37337, 37339, 37357, 37361, 37363, 37369, 37379, 37397,
+    37409, 37423, 37441, 37447, 37463, 37483, 37489, 37493, 37501,
+    37507, 37511, 37517, 37529, 37537, 37547, 37549, 37561, 37567,
+    37571, 37573, 37579, 37589, 37591, 37607, 37619, 37633, 37643,
+    37649, 37657, 37663, 37691, 37693, 37699, 37717, 37747, 37781,
+    37783, 37799, 37811, 37813, 37831, 37847, 37853, 37861, 37871,
+    37879, 37889, 37897, 37907, 37951, 37957, 37963, 37967, 37987,
+    37991, 37993, 37997, 38011, 38039, 38047, 38053, 38069, 38083,
+    38113, 38119, 38149, 38153, 38167, 38177, 38183, 38189, 38197,
+    38201, 38219, 38231, 38237, 38239, 38261, 38273, 38281, 38287,
+    38299, 38303, 38317, 38321, 38327, 38329, 38333, 38351, 38371,
+    38377, 38393, 38431, 38447, 38449, 38453, 38459, 38461, 38501,
+    38543, 38557, 38561, 38567, 38569, 38593, 38603, 38609, 38611,
+    38629, 38639, 38651, 38653, 38669, 38671, 38677, 38693, 38699,
+    38707, 38711, 38713, 38723, 38729, 38737, 38747, 38749, 38767,
+    38783, 38791, 38803, 38821, 38833, 38839, 38851, 38861, 38867,
+    38873, 38891, 38903, 38917, 38921, 38923, 38933, 38953, 38959,
+    38971, 38977, 38993, 39019, 39023, 39041, 39043, 39047, 39079,
+    39089, 39097, 39103, 39107, 39113, 39119, 39133, 39139, 39157,
+    39161, 39163, 39181, 39191, 39199, 39209, 39217, 39227, 39229,
+    39233, 39239, 39241, 39251, 39293, 39301, 39313, 39317, 39323,
+    39341, 39343, 39359, 39367, 39371, 39373, 39383, 39397, 39409,
+    39419, 39439, 39443, 39451, 39461, 39499, 39503, 39509, 39511,
+    39521, 39541, 39551, 39563, 39569, 39581, 39607, 39619, 39623,
+    39631, 39659, 39667, 39671, 39679, 39703, 39709, 39719, 39727,
+    39733, 39749, 39761, 39769, 39779, 39791, 39799, 39821, 39827,
+    39829, 39839, 39841, 39847, 39857, 39863, 39869, 39877, 39883,
+    39887, 39901, 39929, 39937, 39953, 39971, 39979, 39983, 39989,
+    40009, 40013, 40031, 40037, 40039, 40063, 40087, 40093, 40099,
+    40111, 40123, 40127, 40129, 40151, 40153, 40163, 40169, 40177,
+    40189, 40193, 40213, 40231, 40237, 40241, 40253, 40277, 40283,
+    40289, 40343, 40351, 40357, 40361, 40387, 40423, 40427, 40429,
+    40433, 40459, 40471, 40483, 40487, 40493, 40499, 40507, 40519,
+    40529, 40531, 40543, 40559, 40577, 40583, 40591, 40597, 40609,
+    40627, 40637, 40639, 40693, 40697, 40699, 40709, 40739, 40751,
+    40759, 40763, 40771, 40787, 40801, 40813, 40819, 40823, 40829,
+    40841, 40847, 40849, 40853, 40867, 40879, 40883, 40897, 40903,
+    40927, 40933, 40939, 40949, 40961, 40973, 40993, 41011, 41017,
+    41023, 41039, 41047, 41051, 41057, 41077, 41081, 41113, 41117,
+    41131, 41141, 41143, 41149, 41161, 41177, 41179, 41183, 41189,
+    41201, 41203, 41213, 41221, 41227, 41231, 41233, 41243, 41257,
+    41263, 41269, 41281, 41299, 41333, 41341, 41351, 41357, 41381,
+    41387, 41389, 41399, 41411, 41413, 41443, 41453, 41467, 41479,
+    41491, 41507, 41513, 41519, 41521, 41539, 41543, 41549, 41579,
+    41593, 41597, 41603, 41609, 41611, 41617, 41621, 41627, 41641,
+    41647, 41651, 41659, 41669, 41681, 41687, 41719, 41729, 41737,
+    41759, 41761, 41771, 41777, 41801, 41809, 41813, 41843, 41849,
+    41851, 41863, 41879, 41887, 41893, 41897, 41903, 41911, 41927,
+    41941, 41947, 41953, 41957, 41959, 41969, 41981, 41983, 41999,
+    42013, 42017, 42019, 42023, 42043, 42061, 42071, 42073, 42083,
+    42089, 42101, 42131, 42139, 42157, 42169, 42179, 42181, 42187,
+    42193, 42197, 42209, 42221, 42223, 42227, 42239, 42257, 42281,
+    42283, 42293, 42299, 42307, 42323, 42331, 42337, 42349, 42359,
+    42373, 42379, 42391, 42397, 42403, 42407, 42409, 42433, 42437,
+    42443, 42451, 42457, 42461, 42463, 42467, 42473, 42487, 42491,
+    42499, 42509, 42533, 42557, 42569, 42571, 42577, 42589, 42611,
+    42641, 42643, 42649, 42667, 42677, 42683, 42689, 42697, 42701,
+    42703, 42709, 42719, 42727, 42737, 42743, 42751, 42767, 42773,
+    42787, 42793, 42797, 42821, 42829, 42839, 42841, 42853, 42859,
+    42863, 42899, 42901, 42923, 42929, 42937, 42943, 42953, 42961,
+    42967, 42979, 42989, 43003, 43013, 43019, 43037, 43049, 43051,
+    43063, 43067, 43093, 43103, 43117, 43133, 43151, 43159, 43177,
+    43189, 43201, 43207, 43223, 43237, 43261, 43271, 43283, 43291,
+    43313, 43319, 43321, 43331, 43391, 43397, 43399, 43403, 43411,
+    43427, 43441, 43451, 43457, 43481, 43487, 43499, 43517, 43541,
+    43543, 43573, 43577, 43579, 43591, 43597, 43607, 43609, 43613,
+    43627, 43633, 43649, 43651, 43661, 43669, 43691, 43711, 43717,
+    43721, 43753, 43759, 43777, 43781, 43783, 43787, 43789, 43793,
+    43801, 43853, 43867, 43889, 43891, 43913, 43933, 43943, 43951,
+    43961, 43963, 43969, 43973, 43987, 43991, 43997, 44017, 44021,
+    44027, 44029, 44041, 44053, 44059, 44071, 44087, 44089, 44101,
+    44111, 44119, 44123, 44129, 44131, 44159, 44171, 44179, 44189,
+    44201, 44203, 44207, 44221, 44249, 44257, 44263, 44267, 44269,
+    44273, 44279, 44281, 44293, 44351, 44357, 44371, 44381, 44383,
+    44389, 44417, 44449, 44453, 44483, 44491, 44497, 44501, 44507,
+    44519, 44531, 44533, 44537, 44543, 44549, 44563, 44579, 44587,
+    44617, 44621, 44623, 44633, 44641, 44647, 44651, 44657, 44683,
+    44687, 44699, 44701, 44711, 44729, 44741, 44753, 44771, 44773,
+    44777, 44789, 44797, 44809, 44819, 44839, 44843, 44851, 44867,
+    44879, 44887, 44893, 44909, 44917, 44927, 44939, 44953, 44959,
+    44963, 44971, 44983, 44987, 45007, 45013, 45053, 45061, 45077,
+    45083, 45119, 45121, 45127, 45131, 45137, 45139, 45161, 45179,
+    45181, 45191, 45197, 45233, 45247, 45259, 45263, 45281, 45289,
+    45293, 45307, 45317, 45319, 45329, 45337, 45341, 45343, 45361,
+    45377, 45389, 45403, 45413, 45427, 45433, 45439, 45481, 45491,
+    45497, 45503, 45523, 45533, 45541, 45553, 45557, 45569, 45587,
+    45589, 45599, 45613, 45631, 45641, 45659, 45667, 45673, 45677,
+    45691, 45697, 45707, 45737, 45751, 45757, 45763, 45767, 45779,
+    45817, 45821, 45823, 45827, 45833, 45841, 45853, 45863, 45869,
+    45887, 45893, 45943, 45949, 45953, 45959, 45971, 45979, 45989,
+    46021, 46027, 46049, 46051, 46061, 46073, 46091, 46093, 46099,
+    46103, 46133, 46141, 46147, 46153, 46171, 46181, 46183, 46187,
+    46199, 46219, 46229, 46237, 46261, 46271, 46273, 46279, 46301,
+    46307, 46309, 46327, 46337, 46349, 46351, 46381, 46399, 46411,
+    46439, 46441, 46447, 46451, 46457, 46471, 46477, 46489, 46499,
+    46507, 46511, 46523, 46549, 46559, 46567, 46573, 46589, 46591,
+    46601, 46619, 46633, 46639, 46643, 46649, 46663, 46679, 46681,
+    46687, 46691, 46703, 46723, 46727, 46747, 46751, 46757, 46769,
+    46771, 46807, 46811, 46817, 46819, 46829, 46831, 46853, 46861,
+    46867, 46877, 46889, 46901, 46919, 46933, 46957, 46993, 46997,
+    47017, 47041, 47051, 47057, 47059, 47087, 47093, 47111, 47119,
+    47123, 47129, 47137, 47143, 47147, 47149, 47161, 47189, 47207,
+    47221, 47237, 47251, 47269, 47279, 47287, 47293, 47297, 47303,
+    47309, 47317, 47339, 47351, 47353, 47363, 47381, 47387, 47389,
+    47407, 47417, 47419, 47431, 47441, 47459, 47491, 47497, 47501,
+    47507, 47513, 47521, 47527, 47533, 47543, 47563, 47569, 47581,
+    47591, 47599, 47609, 47623, 47629, 47639, 47653, 47657, 47659,
+    47681, 47699, 47701, 47711, 47713, 47717, 47737, 47741, 47743,
+    47777, 47779, 47791, 47797, 47807, 47809, 47819, 47837, 47843,
+    47857, 47869, 47881, 47903, 47911, 47917, 47933, 47939, 47947,
+    47951, 47963, 47969, 47977, 47981, 48017, 48023, 48029, 48049,
+    48073, 48079, 48091, 48109, 48119, 48121, 48131, 48157, 48163,
+    48179, 48187, 48193, 48197, 48221, 48239, 48247, 48259, 48271,
+    48281, 48299, 48311, 48313, 48337, 48341, 48353, 48371, 48383,
+    48397, 48407, 48409, 48413, 48437, 48449, 48463, 48473, 48479,
+    48481, 48487, 48491, 48497, 48523, 48527, 48533, 48539, 48541,
+    48563, 48571, 48589, 48593, 48611, 48619, 48623, 48647, 48649,
+    48661, 48673, 48677, 48679, 48731, 48733, 48751, 48757, 48761,
+    48767, 48779, 48781, 48787, 48799, 48809, 48817, 48821, 48823,
+    48847, 48857, 48859, 48869, 48871, 48883, 48889, 48907, 48947,
+    48953, 48973, 48989, 48991, 49003, 49009, 49019, 49031, 49033,
+    49037, 49043, 49057, 49069, 49081, 49103, 49109, 49117, 49121,
+    49123, 49139, 49157, 49169, 49171, 49177, 49193, 49199, 49201,
+    49207, 49211, 49223, 49253, 49261, 49277, 49279, 49297, 49307,
+    49331, 49333, 49339, 49363, 49367, 49369, 49391, 49393, 49409,
+    49411, 49417, 49429, 49433, 49451, 49459, 49463, 49477, 49481,
+    49499, 49523, 49529, 49531, 49537, 49547, 49549, 49559, 49597,
+    49603, 49613, 49627, 49633, 49639, 49663, 49667, 49669, 49681,
+    49697, 49711, 49727, 49739, 49741, 49747, 49757, 49783, 49787,
+    49789, 49801, 49807, 49811, 49823, 49831, 49843, 49853, 49871,
+    49877, 49891, 49919, 49921, 49927, 49937, 49939, 49943, 49957,
+    49991, 49993, 49999, 50021, 50023, 50033, 50047, 50051, 50053,
+    50069, 50077, 50087, 50093, 50101, 50111, 50119, 50123, 50129,
+    50131, 50147, 50153, 50159, 50177, 50207, 50221, 50227, 50231,
+    50261, 50263, 50273, 50287, 50291, 50311, 50321, 50329, 50333,
+    50341, 50359, 50363, 50377, 50383, 50387, 50411, 50417, 50423,
+    50441, 50459, 50461, 50497, 50503, 50513, 50527, 50539, 50543,
+    50549, 50551, 50581, 50587, 50591, 50593, 50599, 50627, 50647,
+    50651, 50671, 50683, 50707, 50723, 50741, 50753, 50767, 50773,
+    50777, 50789, 50821, 50833, 50839, 50849, 50857, 50867, 50873,
+    50891, 50893, 50909, 50923, 50929, 50951, 50957, 50969, 50971,
+    50989, 50993, 51001, 51031, 51043, 51047, 51059, 51061, 51071,
+    51109, 51131, 51133, 51137, 51151, 51157, 51169, 51193, 51197,
+    51199, 51203, 51217, 51229, 51239, 51241, 51257, 51263, 51283,
+    51287, 51307, 51329, 51341, 51343, 51347, 51349, 51361, 51383,
+    51407, 51413, 51419, 51421, 51427, 51431, 51437, 51439, 51449,
+    51461, 51473, 51479, 51481, 51487, 51503, 51511, 51517, 51521,
+    51539, 51551, 51563, 51577, 51581, 51593, 51599, 51607, 51613,
+    51631, 51637, 51647, 51659, 51673, 51679, 51683, 51691, 51713,
+    51719, 51721, 51749, 51767, 51769, 51787, 51797, 51803, 51817,
+    51827, 51829, 51839, 51853, 51859, 51869, 51871, 51893, 51899,
+    51907, 51913, 51929, 51941, 51949, 51971, 51973, 51977, 51991,
+    52009, 52021, 52027, 52051, 52057, 52067, 52069, 52081, 52103,
+    52121, 52127, 52147, 52153, 52163, 52177, 52181, 52183, 52189,
+    52201, 52223, 52237, 52249, 52253, 52259, 52267, 52289, 52291,
+    52301, 52313, 52321, 52361, 52363, 52369, 52379, 52387, 52391,
+    52433, 52453, 52457, 52489, 52501, 52511, 52517, 52529, 52541,
+    52543, 52553, 52561, 52567, 52571, 52579, 52583, 52609, 52627,
+    52631, 52639, 52667, 52673, 52691, 52697, 52709, 52711, 52721,
+    52727, 52733, 52747, 52757, 52769, 52783, 52807, 52813, 52817,
+    52837, 52859, 52861, 52879, 52883, 52889, 52901, 52903, 52919,
+    52937, 52951, 52957, 52963, 52967, 52973, 52981, 52999, 53003,
+    53017, 53047, 53051, 53069, 53077, 53087, 53089, 53093, 53101,
+    53113, 53117, 53129, 53147, 53149, 53161, 53171, 53173, 53189,
+    53197, 53201, 53231, 53233, 53239, 53267, 53269, 53279, 53281,
+    53299, 53309, 53323, 53327, 53353, 53359, 53377, 53381, 53401,
+    53407, 53411, 53419, 53437, 53441, 53453, 53479, 53503, 53507,
+    53527, 53549, 53551, 53569, 53591, 53593, 53597, 53609, 53611,
+    53617, 53623, 53629, 53633, 53639, 53653, 53657, 53681, 53693,
+    53699, 53717, 53719, 53731, 53759, 53773, 53777, 53783, 53791,
+    53813, 53819, 53831, 53849, 53857, 53861, 53881, 53887, 53891,
+    53897, 53899, 53917, 53923, 53927, 53939, 53951, 53959, 53987,
+    53993, 54001, 54011, 54013, 54037, 54049, 54059, 54083, 54091,
+    54101, 54121, 54133, 54139, 54151, 54163, 54167, 54181, 54193,
+    54217, 54251, 54269, 54277, 54287, 54293, 54311, 54319, 54323,
+    54331, 54347, 54361, 54367, 54371, 54377, 54401, 54403, 54409,
+    54413, 54419, 54421, 54437, 54443, 54449, 54469, 54493, 54497,
+    54499, 54503, 54517, 54521, 54539, 54541, 54547, 54559, 54563,
+    54577, 54581, 54583, 54601, 54617, 54623, 54629, 54631, 54647,
+    54667, 54673, 54679, 54709, 54713, 54721, 54727, 54751, 54767,
+    54773, 54779, 54787, 54799, 54829, 54833, 54851, 54869, 54877,
+    54881, 54907, 54917, 54919, 54941, 54949, 54959, 54973, 54979,
+    54983, 55001, 55009, 55021, 55049, 55051, 55057, 55061, 55073,
+    55079, 55103, 55109, 55117, 55127, 55147, 55163, 55171, 55201,
+    55207, 55213, 55217, 55219, 55229, 55243, 55249, 55259, 55291,
+    55313, 55331, 55333, 55337, 55339, 55343, 55351, 55373, 55381,
+    55399, 55411, 55439, 55441, 55457, 55469, 55487, 55501, 55511,
+    55529, 55541, 55547, 55579, 55589, 55603, 55609, 55619, 55621,
+    55631, 55633, 55639, 55661, 55663, 55667, 55673, 55681, 55691,
+    55697, 55711, 55717, 55721, 55733, 55763, 55787, 55793, 55799,
+    55807, 55813, 55817, 55819, 55823, 55829, 55837, 55843, 55849,
+    55871, 55889, 55897, 55901, 55903, 55921, 55927, 55931, 55933,
+    55949, 55967, 55987, 55997, 56003, 56009, 56039, 56041, 56053,
+    56081, 56087, 56093, 56099, 56101, 56113, 56123, 56131, 56149,
+    56167, 56171, 56179, 56197, 56207, 56209, 56237, 56239, 56249,
+    56263, 56267, 56269, 56299, 56311, 56333, 56359, 56369, 56377,
+    56383, 56393, 56401, 56417, 56431, 56437, 56443, 56453, 56467,
+    56473, 56477, 56479, 56489, 56501, 56503, 56509, 56519, 56527,
+    56531, 56533, 56543, 56569, 56591, 56597, 56599, 56611, 56629,
+    56633, 56659, 56663, 56671, 56681, 56687, 56701, 56711, 56713,
+    56731, 56737, 56747, 56767, 56773, 56779, 56783, 56807, 56809,
+    56813, 56821, 56827, 56843, 56857, 56873, 56891, 56893, 56897,
+    56909, 56911, 56921, 56923, 56929, 56941, 56951, 56957, 56963,
+    56983, 56989, 56993, 56999, 57037, 57041, 57047, 57059, 57073,
+    57077, 57089, 57097, 57107, 57119, 57131, 57139, 57143, 57149,
+    57163, 57173, 57179, 57191, 57193, 57203, 57221, 57223, 57241,
+    57251, 57259, 57269, 57271, 57283, 57287, 57301, 57329, 57331,
+    57347, 57349, 57367, 57373, 57383, 57389, 57397, 57413, 57427,
+    57457, 57467, 57487, 57493, 57503, 57527, 57529, 57557, 57559,
+    57571, 57587, 57593, 57601, 57637, 57641, 57649, 57653, 57667,
+    57679, 57689, 57697, 57709, 57713, 57719, 57727, 57731, 57737,
+    57751, 57773, 57781, 57787, 57791, 57793, 57803, 57809, 57829,
+    57839, 57847, 57853, 57859, 57881, 57899, 57901, 57917, 57923,
+    57943, 57947, 57973, 57977, 57991, 58013, 58027, 58031, 58043,
+    58049, 58057, 58061, 58067, 58073, 58099, 58109, 58111, 58129,
+    58147, 58151, 58153, 58169, 58171, 58189, 58193, 58199, 58207,
+    58211, 58217, 58229, 58231, 58237, 58243, 58271, 58309, 58313,
+    58321, 58337, 58363, 58367, 58369, 58379, 58391, 58393, 58403,
+    58411, 58417, 58427, 58439, 58441, 58451, 58453, 58477, 58481,
+    58511, 58537, 58543, 58549, 58567, 58573, 58579, 58601, 58603,
+    58613, 58631, 58657, 58661, 58679, 58687, 58693, 58699, 58711,
+    58727, 58733, 58741, 58757, 58763, 58771, 58787, 58789, 58831,
+    58889, 58897, 58901, 58907, 58909, 58913, 58921, 58937, 58943,
+    58963, 58967, 58979, 58991, 58997, 59009, 59011, 59021, 59023,
+    59029, 59051, 59053, 59063, 59069, 59077, 59083, 59093, 59107,
+    59113, 59119, 59123, 59141, 59149, 59159, 59167, 59183, 59197,
+    59207, 59209, 59219, 59221, 59233, 59239, 59243, 59263, 59273,
+    59281, 59333, 59341, 59351, 59357, 59359, 59369, 59377, 59387,
+    59393, 59399, 59407, 59417, 59419, 59441, 59443, 59447, 59453,
+    59467, 59471, 59473, 59497, 59509, 59513, 59539, 59557, 59561,
+    59567, 59581, 59611, 59617, 59621, 59627, 59629, 59651, 59659,
+    59663, 59669, 59671, 59693, 59699, 59707, 59723, 59729, 59743,
+    59747, 59753, 59771, 59779, 59791, 59797, 59809, 59833, 59863,
+    59879, 59887, 59921, 59929, 59951, 59957, 59971, 59981, 59999,
+    60013, 60017, 60029, 60037, 60041, 60077, 60083, 60089, 60091,
+    60101, 60103, 60107, 60127, 60133, 60139, 60149, 60161, 60167,
+    60169, 60209, 60217, 60223, 60251, 60257, 60259, 60271, 60289,
+    60293, 60317, 60331, 60337, 60343, 60353, 60373, 60383, 60397,
+    60413, 60427, 60443, 60449, 60457, 60493, 60497, 60509, 60521,
+    60527, 60539, 60589, 60601, 60607, 60611, 60617, 60623, 60631,
+    60637, 60647, 60649, 60659, 60661, 60679, 60689, 60703, 60719,
+    60727, 60733, 60737, 60757, 60761, 60763, 60773, 60779, 60793,
+    60811, 60821, 60859, 60869, 60887, 60889, 60899, 60901, 60913,
+    60917, 60919, 60923, 60937, 60943, 60953, 60961, 61001, 61007,
+    61027, 61031, 61043, 61051, 61057, 61091, 61099, 61121, 61129,
+    61141, 61151, 61153, 61169, 61211, 61223, 61231, 61253, 61261,
+    61283, 61291, 61297, 61331, 61333, 61339, 61343, 61357, 61363,
+    61379, 61381, 61403, 61409, 61417, 61441, 61463, 61469, 61471,
+    61483, 61487, 61493, 61507, 61511, 61519, 61543, 61547, 61553,
+    61559, 61561, 61583, 61603, 61609, 61613, 61627, 61631, 61637,
+    61643, 61651, 61657, 61667, 61673, 61681, 61687, 61703, 61717,
+    61723, 61729, 61751, 61757, 61781, 61813, 61819, 61837, 61843,
+    61861, 61871, 61879, 61909, 61927, 61933, 61949, 61961, 61967,
+    61979, 61981, 61987, 61991, 62003, 62011, 62017, 62039, 62047,
+    62053, 62057, 62071, 62081, 62099, 62119, 62129, 62131, 62137,
+    62141, 62143, 62171, 62189, 62191, 62201, 62207, 62213, 62219,
+    62233, 62273, 62297, 62299, 62303, 62311, 62323, 62327, 62347,
+    62351, 62383, 62401, 62417, 62423, 62459, 62467, 62473, 62477,
+    62483, 62497, 62501, 62507, 62533, 62539, 62549, 62563, 62581,
+    62591, 62597, 62603, 62617, 62627, 62633, 62639, 62653, 62659,
+    62683, 62687, 62701, 62723, 62731, 62743, 62753, 62761, 62773,
+    62791, 62801, 62819, 62827, 62851, 62861, 62869, 62873, 62897,
+    62903, 62921, 62927, 62929, 62939, 62969, 62971, 62981, 62983,
+    62987, 62989, 63029, 63031, 63059, 63067, 63073, 63079, 63097,
+    63103, 63113, 63127, 63131, 63149, 63179, 63197, 63199, 63211,
+    63241, 63247, 63277, 63281, 63299, 63311, 63313, 63317, 63331,
+    63337, 63347, 63353, 63361, 63367, 63377, 63389, 63391, 63397,
+    63409, 63419, 63421, 63439, 63443, 63463, 63467, 63473, 63487,
+    63493, 63499, 63521, 63527, 63533, 63541, 63559, 63577, 63587,
+    63589, 63599, 63601, 63607, 63611, 63617, 63629, 63647, 63649,
+    63659, 63667, 63671, 63689, 63691, 63697, 63703, 63709, 63719,
+    63727, 63737, 63743, 63761, 63773, 63781, 63793, 63799, 63803,
+    63809, 63823, 63839, 63841, 63853, 63857, 63863, 63901, 63907,
+    63913, 63929, 63949, 63977, 63997, 64007, 64013, 64019, 64033,
+    64037, 64063, 64067, 64081, 64091, 64109, 64123, 64151, 64153,
+    64157, 64171, 64187, 64189, 64217, 64223, 64231, 64237, 64271,
+    64279, 64283, 64301, 64303, 64319, 64327, 64333, 64373, 64381,
+    64399, 64403, 64433, 64439, 64451, 64453, 64483, 64489, 64499,
+    64513, 64553, 64567, 64577, 64579, 64591, 64601, 64609, 64613,
+    64621, 64627, 64633, 64661, 64663, 64667, 64679, 64693, 64709,
+    64717, 64747, 64763, 64781, 64783, 64793, 64811, 64817, 64849,
+    64853, 64871, 64877, 64879, 64891, 64901, 64919, 64921, 64927,
+    64937, 64951, 64969, 64997, 65003, 65011, 65027, 65029, 65033,
+    65053, 65063, 65071, 65089, 65099, 65101, 65111, 65119, 65123,
+    65129, 65141, 65147, 65167, 65171, 65173, 65179, 65183, 65203,
+    65213, 65239, 65257, 65267, 65269, 65287, 65293, 65309, 65323,
+    65327, 65353, 65357, 65371, 65381, 65393, 65407, 65413, 65419,
+    65423, 65437, 65447, 65449, 65479, 65497, 65519, 65521, 65537,
+    65539, 65543, 65551, 65557, 65563, 65579, 65581, 65587, 65599,
+    65609, 65617, 65629, 65633, 65647, 65651, 65657, 65677, 65687,
+    65699, 65701, 65707, 65713, 65717, 65719, 65729, 65731, 65761,
+    65777, 65789, 65809, 65827, 65831, 65837, 65839, 65843, 65851,
+    65867, 65881, 65899, 65921, 65927, 65929, 65951, 65957, 65963,
+    65981, 65983, 65993, 66029, 66037, 66041, 66047, 66067, 66071,
+    66083, 66089, 66103, 66107, 66109, 66137, 66161, 66169, 66173,
+    66179, 66191, 66221, 66239, 66271, 66293, 66301, 66337, 66343,
+    66347, 66359, 66361, 66373, 66377, 66383, 66403, 66413, 66431,
+    66449, 66457, 66463, 66467, 66491, 66499, 66509, 66523, 66529,
+    66533, 66541, 66553, 66569, 66571, 66587, 66593, 66601, 66617,
+    66629, 66643, 66653, 66683, 66697, 66701, 66713, 66721, 66733,
+    66739, 66749, 66751, 66763, 66791, 66797, 66809, 66821, 66841,
+    66851, 66853, 66863, 66877, 66883, 66889, 66919, 66923, 66931,
+    66943, 66947, 66949, 66959, 66973, 66977, 67003, 67021, 67033,
+    67043, 67049, 67057, 67061, 67073, 67079, 67103, 67121, 67129,
+    67139, 67141, 67153, 67157, 67169, 67181, 67187, 67189, 67211,
+    67213, 67217, 67219, 67231, 67247, 67261, 67271, 67273, 67289,
+    67307, 67339, 67343, 67349, 67369, 67391, 67399, 67409, 67411,
+    67421, 67427, 67429, 67433, 67447, 67453, 67477, 67481, 67489,
+    67493, 67499, 67511, 67523, 67531, 67537, 67547, 67559, 67567,
+    67577, 67579, 67589, 67601, 67607, 67619, 67631, 67651, 67679,
+    67699, 67709, 67723, 67733, 67741, 67751, 67757, 67759, 67763,
+    67777, 67783, 67789, 67801, 67807, 67819, 67829, 67843, 67853,
+    67867, 67883, 67891, 67901, 67927, 67931, 67933, 67939, 67943,
+    67957, 67961, 67967, 67979, 67987, 67993, 68023, 68041, 68053,
+    68059, 68071, 68087, 68099, 68111, 68113, 68141, 68147, 68161,
+    68171, 68207, 68209, 68213, 68219, 68227, 68239, 68261, 68279,
+    68281, 68311, 68329, 68351, 68371, 68389, 68399, 68437, 68443,
+    68447, 68449, 68473, 68477, 68483, 68489, 68491, 68501, 68507,
+    68521, 68531, 68539, 68543, 68567, 68581, 68597, 68611, 68633,
+    68639, 68659, 68669, 68683, 68687, 68699, 68711, 68713, 68729,
+    68737, 68743, 68749, 68767, 68771, 68777, 68791, 68813, 68819,
+    68821, 68863, 68879, 68881, 68891, 68897, 68899, 68903, 68909,
+    68917, 68927, 68947, 68963, 68993, 69001, 69011, 69019, 69029,
+    69031, 69061, 69067, 69073, 69109, 69119, 69127, 69143, 69149,
+    69151, 69163, 69191, 69193, 69197, 69203, 69221, 69233, 69239,
+    69247, 69257, 69259, 69263, 69313, 69317, 69337, 69341, 69371,
+    69379, 69383, 69389, 69401, 69403, 69427, 69431, 69439, 69457,
+    69463, 69467, 69473, 69481, 69491, 69493, 69497, 69499, 69539,
+    69557, 69593, 69623, 69653, 69661, 69677, 69691, 69697, 69709,
+    69737, 69739, 69761, 69763, 69767, 69779, 69809, 69821, 69827,
+    69829, 69833, 69847, 69857, 69859, 69877, 69899, 69911, 69929,
+    69931, 69941, 69959, 69991, 69997, 70001, 70003, 70009, 70019,
+    70039, 70051, 70061, 70067, 70079, 70099, 70111, 70117, 70121,
+    70123, 70139, 70141, 70157, 70163, 70177, 70181, 70183, 70199,
+    70201, 70207, 70223, 70229, 70237, 70241, 70249, 70271, 70289,
+    70297, 70309, 70313, 70321, 70327, 70351, 70373, 70379, 70381,
+    70393, 70423, 70429, 70439, 70451, 70457, 70459, 70481, 70487,
+    70489, 70501, 70507, 70529, 70537, 70549, 70571, 70573, 70583,
+    70589, 70607, 70619, 70621, 70627, 70639, 70657, 70663, 70667,
+    70687, 70709, 70717, 70729, 70753, 70769, 70783, 70793, 70823,
+    70841, 70843, 70849, 70853, 70867, 70877, 70879, 70891, 70901,
+    70913, 70919, 70921, 70937, 70949, 70951, 70957, 70969, 70979,
+    70981, 70991, 70997, 70999, 71011, 71023, 71039, 71059, 71069,
+    71081, 71089, 71119, 71129, 71143, 71147, 71153, 71161, 71167,
+    71171, 71191, 71209, 71233, 71237, 71249, 71257, 71261, 71263,
+    71287, 71293, 71317, 71327, 71329, 71333, 71339, 71341, 71347,
+    71353, 71359, 71363, 71387, 71389, 71399, 71411, 71413, 71419,
+    71429, 71437, 71443, 71453, 71471, 71473, 71479, 71483, 71503,
+    71527, 71537, 71549, 71551, 71563, 71569, 71593, 71597, 71633,
+    71647, 71663, 71671, 71693, 71699, 71707, 71711, 71713, 71719,
+    71741, 71761, 71777, 71789, 71807, 71809, 71821, 71837, 71843,
+    71849, 71861, 71867, 71879, 71881, 71887, 71899, 71909, 71917,
+    71933, 71941, 71947, 71963, 71971, 71983, 71987, 71993, 71999,
+    72019, 72031, 72043, 72047, 72053, 72073, 72077, 72089, 72091,
+    72101, 72103, 72109, 72139, 72161, 72167, 72169, 72173, 72211,
+    72221, 72223, 72227, 72229, 72251, 72253, 72269, 72271, 72277,
+    72287, 72307, 72313, 72337, 72341, 72353, 72367, 72379, 72383,
+    72421, 72431, 72461, 72467, 72469, 72481, 72493, 72497, 72503,
+    72533, 72547, 72551, 72559, 72577, 72613, 72617, 72623, 72643,
+    72647, 72649, 72661, 72671, 72673, 72679, 72689, 72701, 72707,
+    72719, 72727, 72733, 72739, 72763, 72767, 72797, 72817, 72823,
+    72859, 72869, 72871, 72883, 72889, 72893, 72901, 72907, 72911,
+    72923, 72931, 72937, 72949, 72953, 72959, 72973, 72977, 72997,
+    73009, 73013, 73019, 73037, 73039, 73043, 73061, 73063, 73079,
+    73091, 73121, 73127, 73133, 73141, 73181, 73189, 73237, 73243,
+    73259, 73277, 73291, 73303, 73309, 73327, 73331, 73351, 73361,
+    73363, 73369, 73379, 73387, 73417, 73421, 73433, 73453, 73459,
+    73471, 73477, 73483, 73517, 73523, 73529, 73547, 73553, 73561,
+    73571, 73583, 73589, 73597, 73607, 73609, 73613, 73637, 73643,
+    73651, 73673, 73679, 73681, 73693, 73699, 73709, 73721, 73727,
+    73751, 73757, 73771, 73783, 73819, 73823, 73847, 73849, 73859,
+    73867, 73877, 73883, 73897, 73907, 73939, 73943, 73951, 73961,
+    73973, 73999, 74017, 74021, 74027, 74047, 74051, 74071, 74077,
+    74093, 74099, 74101, 74131, 74143, 74149, 74159, 74161, 74167,
+    74177, 74189, 74197, 74201, 74203, 74209, 74219, 74231, 74257,
+    74279, 74287, 74293, 74297, 74311, 74317, 74323, 74353, 74357,
+    74363, 74377, 74381, 74383, 74411, 74413, 74419, 74441, 74449,
+    74453, 74471, 74489, 74507, 74509, 74521, 74527, 74531, 74551,
+    74561, 74567, 74573, 74587, 74597, 74609, 74611, 74623, 74653,
+    74687, 74699, 74707, 74713, 74717, 74719, 74729, 74731, 74747,
+    74759, 74761, 74771, 74779, 74797, 74821, 74827, 74831, 74843,
+    74857, 74861, 74869, 74873, 74887, 74891, 74897, 74903, 74923,
+    74929, 74933, 74941, 74959, 75011, 75013, 75017, 75029, 75037,
+    75041, 75079, 75083, 75109, 75133, 75149, 75161, 75167, 75169,
+    75181, 75193, 75209, 75211, 75217, 75223, 75227, 75239, 75253,
+    75269, 75277, 75289, 75307, 75323, 75329, 75337, 75347, 75353,
+    75367, 75377, 75389, 75391, 75401, 75403, 75407, 75431, 75437,
+    75479, 75503, 75511, 75521, 75527, 75533, 75539, 75541, 75553,
+    75557, 75571, 75577, 75583, 75611, 75617, 75619, 75629, 75641,
+    75653, 75659, 75679, 75683, 75689, 75703, 75707, 75709, 75721,
+    75731, 75743, 75767, 75773, 75781, 75787, 75793, 75797, 75821,
+    75833, 75853, 75869, 75883, 75913, 75931, 75937, 75941, 75967,
+    75979, 75983, 75989, 75991, 75997, 76001, 76003, 76031, 76039,
+    76079, 76081, 76091, 76099, 76103, 76123, 76129, 76147, 76157,
+    76159, 76163, 76207, 76213, 76231, 76243, 76249, 76253, 76259,
+    76261, 76283, 76289, 76303, 76333, 76343, 76367, 76369, 76379,
+    76387, 76403, 76421, 76423, 76441, 76463, 76471, 76481, 76487,
+    76493, 76507, 76511, 76519, 76537, 76541, 76543, 76561, 76579,
+    76597, 76603, 76607, 76631, 76649, 76651, 76667, 76673, 76679,
+    76697, 76717, 76733, 76753, 76757, 76771, 76777, 76781, 76801,
+    76819, 76829, 76831, 76837, 76847, 76871, 76873, 76883, 76907,
+    76913, 76919, 76943, 76949, 76961, 76963, 76991, 77003, 77017,
+    77023, 77029, 77041, 77047, 77069, 77081, 77093, 77101, 77137,
+    77141, 77153, 77167, 77171, 77191, 77201, 77213, 77237, 77239,
+    77243, 77249, 77261, 77263, 77267, 77269, 77279, 77291, 77317,
+    77323, 77339, 77347, 77351, 77359, 77369, 77377, 77383, 77417,
+    77419, 77431, 77447, 77471, 77477, 77479, 77489, 77491, 77509,
+    77513, 77521, 77527, 77543, 77549, 77551, 77557, 77563, 77569,
+    77573, 77587, 77591, 77611, 77617, 77621, 77641, 77647, 77659,
+    77681, 77687, 77689, 77699, 77711, 77713, 77719, 77723, 77731,
+    77743, 77747, 77761, 77773, 77783, 77797, 77801, 77813, 77839,
+    77849, 77863, 77867, 77893, 77899, 77929, 77933, 77951, 77969,
+    77977, 77983, 77999, 78007, 78017, 78031, 78041, 78049, 78059,
+    78079, 78101, 78121, 78137, 78139, 78157, 78163, 78167, 78173,
+    78179, 78191, 78193, 78203, 78229, 78233, 78241, 78259, 78277,
+    78283, 78301, 78307, 78311, 78317, 78341, 78347, 78367, 78401,
+    78427, 78437, 78439, 78467, 78479, 78487, 78497, 78509, 78511,
+    78517, 78539, 78541, 78553, 78569, 78571, 78577, 78583, 78593,
+    78607, 78623, 78643, 78649, 78653, 78691, 78697, 78707, 78713,
+    78721, 78737, 78779, 78781, 78787, 78791, 78797, 78803, 78809,
+    78823, 78839, 78853, 78857, 78877, 78887, 78889, 78893, 78901,
+    78919, 78929, 78941, 78977, 78979, 78989, 79031, 79039, 79043,
+    79063, 79087, 79103, 79111, 79133, 79139, 79147, 79151, 79153,
+    79159, 79181, 79187, 79193, 79201, 79229, 79231, 79241, 79259,
+    79273, 79279, 79283, 79301, 79309, 79319, 79333, 79337, 79349,
+    79357, 79367, 79379, 79393, 79397, 79399, 79411, 79423, 79427,
+    79433, 79451, 79481, 79493, 79531, 79537, 79549, 79559, 79561,
+    79579, 79589, 79601, 79609, 79613, 79621, 79627, 79631, 79633,
+    79657, 79669, 79687, 79691, 79693, 79697, 79699, 79757, 79769,
+    79777, 79801, 79811, 79813, 79817, 79823, 79829, 79841, 79843,
+    79847, 79861, 79867, 79873, 79889, 79901, 79903, 79907, 79939,
+    79943, 79967, 79973, 79979, 79987, 79997, 79999, 80021, 80039,
+    80051, 80071, 80077, 80107, 80111, 80141, 80147, 80149, 80153,
+    80167, 80173, 80177, 80191, 80207, 80209, 80221, 80231, 80233,
+    80239, 80251, 80263, 80273, 80279, 80287, 80309, 80317, 80329,
+    80341, 80347, 80363, 80369, 80387, 80407, 80429, 80447, 80449,
+    80471, 80473, 80489, 80491, 80513, 80527, 80537, 80557, 80567,
+    80599, 80603, 80611, 80621, 80627, 80629, 80651, 80657, 80669,
+    80671, 80677, 80681, 80683, 80687, 80701, 80713, 80737, 80747,
+    80749, 80761, 80777, 80779, 80783, 80789, 80803, 80809, 80819,
+    80831, 80833, 80849, 80863, 80897, 80909, 80911, 80917, 80923,
+    80929, 80933, 80953, 80963, 80989, 81001, 81013, 81017, 81019,
+    81023, 81031, 81041, 81043, 81047, 81049, 81071, 81077, 81083,
+    81097, 81101, 81119, 81131, 81157, 81163, 81173, 81181, 81197,
+    81199, 81203, 81223, 81233, 81239, 81281, 81283, 81293, 81299,
+    81307, 81331, 81343, 81349, 81353, 81359, 81371, 81373, 81401,
+    81409, 81421, 81439, 81457, 81463, 81509, 81517, 81527, 81533,
+    81547, 81551, 81553, 81559, 81563, 81569, 81611, 81619, 81629,
+    81637, 81647, 81649, 81667, 81671, 81677, 81689, 81701, 81703,
+    81707, 81727, 81737, 81749, 81761, 81769, 81773, 81799, 81817,
+    81839, 81847, 81853, 81869, 81883, 81899, 81901, 81919, 81929,
+    81931, 81937, 81943, 81953, 81967, 81971, 81973, 82003, 82007,
+    82009, 82013, 82021, 82031, 82037, 82039, 82051, 82067, 82073,
+    82129, 82139, 82141, 82153, 82163, 82171, 82183, 82189, 82193,
+    82207, 82217, 82219, 82223, 82231, 82237, 82241, 82261, 82267,
+    82279, 82301, 82307, 82339, 82349, 82351, 82361, 82373, 82387,
+    82393, 82421, 82457, 82463, 82469, 82471, 82483, 82487, 82493,
+    82499, 82507, 82529, 82531, 82549, 82559, 82561, 82567, 82571,
+    82591, 82601, 82609, 82613, 82619, 82633, 82651, 82657, 82699,
+    82721, 82723, 82727, 82729, 82757, 82759, 82763, 82781, 82787,
+    82793, 82799, 82811, 82813, 82837, 82847, 82883, 82889, 82891,
+    82903, 82913, 82939, 82963, 82981, 82997, 83003, 83009, 83023,
+    83047, 83059, 83063, 83071, 83077, 83089, 83093, 83101, 83117,
+    83137, 83177, 83203, 83207, 83219, 83221, 83227, 83231, 83233,
+    83243, 83257, 83267, 83269, 83273, 83299, 83311, 83339, 83341,
+    83357, 83383, 83389, 83399, 83401, 83407, 83417, 83423, 83431,
+    83437, 83443, 83449, 83459, 83471, 83477, 83497, 83537, 83557,
+    83561, 83563, 83579, 83591, 83597, 83609, 83617, 83621, 83639,
+    83641, 83653, 83663, 83689, 83701, 83717, 83719, 83737, 83761,
+    83773, 83777, 83791, 83813, 83833, 83843, 83857, 83869, 83873,
+    83891, 83903, 83911, 83921, 83933, 83939, 83969, 83983, 83987,
+    84011, 84017, 84047, 84053, 84059, 84061, 84067, 84089, 84121,
+    84127, 84131, 84137, 84143, 84163, 84179, 84181, 84191, 84199,
+    84211, 84221, 84223, 84229, 84239, 84247, 84263, 84299, 84307,
+    84313, 84317, 84319, 84347, 84349, 84377, 84389, 84391, 84401,
+    84407, 84421, 84431, 84437, 84443, 84449, 84457, 84463, 84467,
+    84481, 84499, 84503, 84509, 84521, 84523, 84533, 84551, 84559,
+    84589, 84629, 84631, 84649, 84653, 84659, 84673, 84691, 84697,
+    84701, 84713, 84719, 84731, 84737, 84751, 84761, 84787, 84793,
+    84809, 84811, 84827, 84857, 84859, 84869, 84871, 84913, 84919,
+    84947, 84961, 84967, 84977, 84979, 84991, 85009, 85021, 85027,
+    85037, 85049, 85061, 85081, 85087, 85091, 85093, 85103, 85109,
+    85121, 85133, 85147, 85159, 85193, 85199, 85201, 85213, 85223,
+    85229, 85237, 85243, 85247, 85259, 85297, 85303, 85313, 85331,
+    85333, 85361, 85363, 85369, 85381, 85411, 85427, 85429, 85439,
+    85447, 85451, 85453, 85469, 85487, 85513, 85517, 85523, 85531,
+    85549, 85571, 85577, 85597, 85601, 85607, 85619, 85621, 85627,
+    85639, 85643, 85661, 85667, 85669, 85691, 85703, 85711, 85717,
+    85733, 85751, 85781, 85793, 85817, 85819, 85829, 85831, 85837,
+    85843, 85847, 85853, 85889, 85903, 85909, 85931, 85933, 85991,
+    85999, 86011, 86017, 86027, 86029, 86069, 86077, 86083, 86111,
+    86113, 86117, 86131, 86137, 86143, 86161, 86171, 86179, 86183,
+    86197, 86201, 86209, 86239, 86243, 86249, 86257, 86263, 86269,
+    86287, 86291, 86293, 86297, 86311, 86323, 86341, 86351, 86353,
+    86357, 86369, 86371, 86381, 86389, 86399, 86413, 86423, 86441,
+    86453, 86461, 86467, 86477, 86491, 86501, 86509, 86531, 86533,
+    86539, 86561, 86573, 86579, 86587, 86599, 86627, 86629, 86677,
+    86689, 86693, 86711, 86719, 86729, 86743, 86753, 86767, 86771,
+    86783, 86813, 86837, 86843, 86851, 86857, 86861, 86869, 86923,
+    86927, 86929, 86939, 86951, 86959, 86969, 86981, 86993, 87011,
+    87013, 87037, 87041, 87049, 87071, 87083, 87103, 87107, 87119,
+    87121, 87133, 87149, 87151, 87179, 87181, 87187, 87211, 87221,
+    87223, 87251, 87253, 87257, 87277, 87281, 87293, 87299, 87313,
+    87317, 87323, 87337, 87359, 87383, 87403, 87407, 87421, 87427,
+    87433, 87443, 87473, 87481, 87491, 87509, 87511, 87517, 87523,
+    87539, 87541, 87547, 87553, 87557, 87559, 87583, 87587, 87589,
+    87613, 87623, 87629, 87631, 87641, 87643, 87649, 87671, 87679,
+    87683, 87691, 87697, 87701, 87719, 87721, 87739, 87743, 87751,
+    87767, 87793, 87797, 87803, 87811, 87833, 87853, 87869, 87877,
+    87881, 87887, 87911, 87917, 87931, 87943, 87959, 87961, 87973,
+    87977, 87991, 88001, 88003, 88007, 88019, 88037, 88069, 88079,
+    88093, 88117, 88129, 88169, 88177, 88211, 88223, 88237, 88241,
+    88259, 88261, 88289, 88301, 88321, 88327, 88337, 88339, 88379,
+    88397, 88411, 88423, 88427, 88463, 88469, 88471, 88493, 88499,
+    88513, 88523, 88547, 88589, 88591, 88607, 88609, 88643, 88651,
+    88657, 88661, 88663, 88667, 88681, 88721, 88729, 88741, 88747,
+    88771, 88789, 88793, 88799, 88801, 88807, 88811, 88813, 88817,
+    88819, 88843, 88853, 88861, 88867, 88873, 88883, 88897, 88903,
+    88919, 88937, 88951, 88969, 88993, 88997, 89003, 89009, 89017,
+    89021, 89041, 89051, 89057, 89069, 89071, 89083, 89087, 89101,
+    89107, 89113, 89119, 89123, 89137, 89153, 89189, 89203, 89209,
+    89213, 89227, 89231, 89237, 89261, 89269, 89273, 89293, 89303,
+    89317, 89329, 89363, 89371, 89381, 89387, 89393, 89399, 89413,
+    89417, 89431, 89443, 89449, 89459, 89477, 89491, 89501, 89513,
+    89519, 89521, 89527, 89533, 89561, 89563, 89567, 89591, 89597,
+    89599, 89603, 89611, 89627, 89633, 89653, 89657, 89659, 89669,
+    89671, 89681, 89689, 89753, 89759, 89767, 89779, 89783, 89797,
+    89809, 89819, 89821, 89833, 89839, 89849, 89867, 89891, 89897,
+    89899, 89909, 89917, 89923, 89939, 89959, 89963, 89977, 89983,
+    89989, 90001, 90007, 90011, 90017, 90019, 90023, 90031, 90053,
+    90059, 90067, 90071, 90073, 90089, 90107, 90121, 90127, 90149,
+    90163, 90173, 90187, 90191, 90197, 90199, 90203, 90217, 90227,
+    90239, 90247, 90263, 90271, 90281, 90289, 90313, 90353, 90359,
+    90371, 90373, 90379, 90397, 90401, 90403, 90407, 90437, 90439,
+    90469, 90473, 90481, 90499, 90511, 90523, 90527, 90529, 90533,
+    90547, 90583, 90599, 90617, 90619, 90631, 90641, 90647, 90659,
+    90677, 90679, 90697, 90703, 90709, 90731, 90749, 90787, 90793,
+    90803, 90821, 90823, 90833, 90841, 90847, 90863, 90887, 90901,
+    90907, 90911, 90917, 90931, 90947, 90971, 90977, 90989, 90997,
+    91009, 91019, 91033, 91079, 91081, 91097, 91099, 91121, 91127,
+    91129, 91139, 91141, 91151, 91153, 91159, 91163, 91183, 91193,
+    91199, 91229, 91237, 91243, 91249, 91253, 91283, 91291, 91297,
+    91303, 91309, 91331, 91367, 91369, 91373, 91381, 91387, 91393,
+    91397, 91411, 91423, 91433, 91453, 91457, 91459, 91463, 91493,
+    91499, 91513, 91529, 91541, 91571, 91573, 91577, 91583, 91591,
+    91621, 91631, 91639, 91673, 91691, 91703, 91711, 91733, 91753,
+    91757, 91771, 91781, 91801, 91807, 91811, 91813, 91823, 91837,
+    91841, 91867, 91873, 91909, 91921, 91939, 91943, 91951, 91957,
+    91961, 91967, 91969, 91997, 92003, 92009, 92033, 92041, 92051,
+    92077, 92083, 92107, 92111, 92119, 92143, 92153, 92173, 92177,
+    92179, 92189, 92203, 92219, 92221, 92227, 92233, 92237, 92243,
+    92251, 92269, 92297, 92311, 92317, 92333, 92347, 92353, 92357,
+    92363, 92369, 92377, 92381, 92383, 92387, 92399, 92401, 92413,
+    92419, 92431, 92459, 92461, 92467, 92479, 92489, 92503, 92507,
+    92551, 92557, 92567, 92569, 92581, 92593, 92623, 92627, 92639,
+    92641, 92647, 92657, 92669, 92671, 92681, 92683, 92693, 92699,
+    92707, 92717, 92723, 92737, 92753, 92761, 92767, 92779, 92789,
+    92791, 92801, 92809, 92821, 92831, 92849, 92857, 92861, 92863,
+    92867, 92893, 92899, 92921, 92927, 92941, 92951, 92957, 92959,
+    92987, 92993, 93001, 93047, 93053, 93059, 93077, 93083, 93089,
+    93097, 93103, 93113, 93131, 93133, 93139, 93151, 93169, 93179,
+    93187, 93199, 93229, 93239, 93241, 93251, 93253, 93257, 93263,
+    93281, 93283, 93287, 93307, 93319, 93323, 93329, 93337, 93371,
+    93377, 93383, 93407, 93419, 93427, 93463, 93479, 93481, 93487,
+    93491, 93493, 93497, 93503, 93523, 93529, 93553, 93557, 93559,
+    93563, 93581, 93601, 93607, 93629, 93637, 93683, 93701, 93703,
+    93719, 93739, 93761, 93763, 93787, 93809, 93811, 93827, 93851,
+    93871, 93887, 93889, 93893, 93901, 93911, 93913, 93923, 93937,
+    93941, 93949, 93967, 93971, 93979, 93983, 93997, 94007, 94009,
+    94033, 94049, 94057, 94063, 94079, 94099, 94109, 94111, 94117,
+    94121, 94151, 94153, 94169, 94201, 94207, 94219, 94229, 94253,
+    94261, 94273, 94291, 94307, 94309, 94321, 94327, 94331, 94343,
+    94349, 94351, 94379, 94397, 94399, 94421, 94427, 94433, 94439,
+    94441, 94447, 94463, 94477, 94483, 94513, 94529, 94531, 94541,
+    94543, 94547, 94559, 94561, 94573, 94583, 94597, 94603, 94613,
+    94621, 94649, 94651, 94687, 94693, 94709, 94723, 94727, 94747,
+    94771, 94777, 94781, 94789, 94793, 94811, 94819, 94823, 94837,
+    94841, 94847, 94849, 94873, 94889, 94903, 94907, 94933, 94949,
+    94951, 94961, 94993, 94999, 95003, 95009, 95021, 95027, 95063,
+    95071, 95083, 95087, 95089, 95093, 95101, 95107, 95111, 95131,
+    95143, 95153, 95177, 95189, 95191, 95203, 95213, 95219, 95231,
+    95233, 95239, 95257, 95261, 95267, 95273, 95279, 95287, 95311,
+    95317, 95327, 95339, 95369, 95383, 95393, 95401, 95413, 95419,
+    95429, 95441, 95443, 95461, 95467, 95471, 95479, 95483, 95507,
+    95527, 95531, 95539, 95549, 95561, 95569, 95581, 95597, 95603,
+    95617, 95621, 95629, 95633, 95651, 95701, 95707, 95713, 95717,
+    95723, 95731, 95737, 95747, 95773, 95783, 95789, 95791, 95801,
+    95803, 95813, 95819, 95857, 95869, 95873, 95881, 95891, 95911,
+    95917, 95923, 95929, 95947, 95957, 95959, 95971, 95987, 95989,
+    96001, 96013, 96017, 96043, 96053, 96059, 96079, 96097, 96137,
+    96149, 96157, 96167, 96179, 96181, 96199, 96211, 96221, 96223,
+    96233, 96259, 96263, 96269, 96281, 96289, 96293, 96323, 96329,
+    96331, 96337, 96353, 96377, 96401, 96419, 96431, 96443, 96451,
+    96457, 96461, 96469, 96479, 96487, 96493, 96497, 96517, 96527,
+    96553, 96557, 96581, 96587, 96589, 96601, 96643, 96661, 96667,
+    96671, 96697, 96703, 96731, 96737, 96739, 96749, 96757, 96763,
+    96769, 96779, 96787, 96797, 96799, 96821, 96823, 96827, 96847,
+    96851, 96857, 96893, 96907, 96911, 96931, 96953, 96959, 96973,
+    96979, 96989, 96997, 97001, 97003, 97007, 97021, 97039, 97073,
+    97081, 97103, 97117, 97127, 97151, 97157, 97159, 97169, 97171,
+    97177, 97187, 97213, 97231, 97241, 97259, 97283, 97301, 97303,
+    97327, 97367, 97369, 97373, 97379, 97381, 97387, 97397, 97423,
+    97429, 97441, 97453, 97459, 97463, 97499, 97501, 97511, 97523,
+    97547, 97549, 97553, 97561, 97571, 97577, 97579, 97583, 97607,
+    97609, 97613, 97649, 97651, 97673, 97687, 97711, 97729, 97771,
+    97777, 97787, 97789, 97813, 97829, 97841, 97843, 97847, 97849,
+    97859, 97861, 97871, 97879, 97883, 97919, 97927, 97931, 97943,
+    97961, 97967, 97973, 97987, 98009, 98011, 98017, 98041, 98047,
+    98057, 98081, 98101, 98123, 98129, 98143, 98179, 98207, 98213,
+    98221, 98227, 98251, 98257, 98269, 98297, 98299, 98317, 98321,
+    98323, 98327, 98347, 98369, 98377, 98387, 98389, 98407, 98411,
+    98419, 98429, 98443, 98453, 98459, 98467, 98473, 98479, 98491,
+    98507, 98519, 98533, 98543, 98561, 98563, 98573, 98597, 98621,
+    98627, 98639, 98641, 98663, 98669, 98689, 98711, 98713, 98717,
+    98729, 98731, 98737, 98773, 98779, 98801, 98807, 98809, 98837,
+    98849, 98867, 98869, 98873, 98887, 98893, 98897, 98899, 98909,
+    98911, 98927, 98929, 98939, 98947, 98953, 98963, 98981, 98993,
+    98999, 99013, 99017, 99023, 99041, 99053, 99079, 99083, 99089,
+    99103, 99109, 99119, 99131, 99133, 99137, 99139, 99149, 99173,
+    99181, 99191, 99223, 99233, 99241, 99251, 99257, 99259, 99277,
+    99289, 99317, 99347, 99349, 99367, 99371, 99377, 99391, 99397,
+    99401, 99409, 99431, 99439, 99469, 99487, 99497, 99523, 99527,
+    99529, 99551, 99559, 99563, 99571, 99577, 99581, 99607, 99611,
+    99623, 99643, 99661, 99667, 99679, 99689, 99707, 99709, 99713,
+    99719, 99721, 99733, 99761, 99767, 99787, 99793, 99809, 99817,
+    99823, 99829, 99833, 99839, 99859, 99871, 99877, 99881, 99901,
+    99907, 99923, 99929, 99961, 99971, 99989, 99991,
+    };
diff --git a/src/rt/isaac/rand.h b/src/rt/isaac/rand.h
new file mode 100644
index 00000000..018496f6
--- /dev/null
+++ b/src/rt/isaac/rand.h
@@ -0,0 +1,56 @@
+/*
+------------------------------------------------------------------------------
+rand.h: definitions for a random number generator
+By Bob Jenkins, 1996, Public Domain
+MODIFIED:
+  960327: Creation (addition of randinit, really)
+  970719: use context, not global variables, for internal state
+  980324: renamed seed to flag
+  980605: recommend RANDSIZL=4 for noncryptography.
+  010626: note this is public domain
+------------------------------------------------------------------------------
+*/
+#ifndef STANDARD
+#include "standard.h"
+#endif
+
+#ifndef RAND
+#define RAND
+#define RANDSIZL   (8)  /* I recommend 8 for crypto, 4 for simulations */
+#define RANDSIZ    (1<<RANDSIZL)
+
+/* context of random number generator */
+struct randctx
+{
+  ub4 randcnt;
+  ub4 randrsl[RANDSIZ];
+  ub4 randmem[RANDSIZ];
+  ub4 randa;
+  ub4 randb;
+  ub4 randc;
+};
+typedef  struct randctx  randctx;
+
+/*
+------------------------------------------------------------------------------
+ If (flag==TRUE), then use the contents of randrsl[0..RANDSIZ-1] as the seed.
+------------------------------------------------------------------------------
+*/
+void randinit(randctx *r, word flag);
+
+void isaac(randctx *r);
+
+
+/*
+------------------------------------------------------------------------------
+ Call rand(/o_ randctx *r _o/) to retrieve a single 32-bit random value
+------------------------------------------------------------------------------
+*/
+#define rand(r) \
+   (!(r)->randcnt-- ? \
+     (isaac(r), (r)->randcnt=RANDSIZ-1, (r)->randrsl[(r)->randcnt]) : \
+     (r)->randrsl[(r)->randcnt])
+
+#endif  /* RAND */
+
+
diff --git a/src/rt/isaac/randport.cpp b/src/rt/isaac/randport.cpp
new file mode 100644
index 00000000..45ec590d
--- /dev/null
+++ b/src/rt/isaac/randport.cpp
@@ -0,0 +1,134 @@
+/*
+------------------------------------------------------------------------------
+rand.c: By Bob Jenkins.  My random number generator, ISAAC.  Public Domain
+MODIFIED:
+  960327: Creation (addition of randinit, really)
+  970719: use context, not global variables, for internal state
+  980324: make a portable version
+  010626: Note this is public domain
+------------------------------------------------------------------------------
+*/
+#ifndef STANDARD
+#include "standard.h"
+#endif
+#ifndef RAND
+#include "rand.h"
+#endif
+
+
+#define ind(mm,x)  ((mm)[(x>>2)&(RANDSIZ-1)])
+#define rngstep(mix,a,b,mm,m,m2,r,x) \
+{ \
+  x = *m;  \
+  a = ((a^(mix)) + *(m2++)) & 0xffffffff; \
+  *(m++) = y = (ind(mm,x) + a + b) & 0xffffffff; \
+  *(r++) = b = (ind(mm,y>>RANDSIZL) + x) & 0xffffffff; \
+}
+
+void     isaac(randctx *ctx)
+{
+   register ub4 a,b,x,y,*m,*mm,*m2,*r,*mend;
+   mm=ctx->randmem; r=ctx->randrsl;
+   a = ctx->randa; b = (ctx->randb + (++ctx->randc)) & 0xffffffff;
+   for (m = mm, mend = m2 = m+(RANDSIZ/2); m<mend; )
+   {
+      rngstep( a<<13, a, b, mm, m, m2, r, x);
+      rngstep( a>>6 , a, b, mm, m, m2, r, x);
+      rngstep( a<<2 , a, b, mm, m, m2, r, x);
+      rngstep( a>>16, a, b, mm, m, m2, r, x);
+   }
+   for (m2 = mm; m2<mend; )
+   {
+      rngstep( a<<13, a, b, mm, m, m2, r, x);
+      rngstep( a>>6 , a, b, mm, m, m2, r, x);
+      rngstep( a<<2 , a, b, mm, m, m2, r, x);
+      rngstep( a>>16, a, b, mm, m, m2, r, x);
+   }
+   ctx->randb = b; ctx->randa = a;
+}
+
+
+#define mix(a,b,c,d,e,f,g,h) \
+{ \
+   a^=b<<11; d+=a; b+=c; \
+   b^=c>>2;  e+=b; c+=d; \
+   c^=d<<8;  f+=c; d+=e; \
+   d^=e>>16; g+=d; e+=f; \
+   e^=f<<10; h+=e; f+=g; \
+   f^=g>>4;  a+=f; g+=h; \
+   g^=h<<8;  b+=g; h+=a; \
+   h^=a>>9;  c+=h; a+=b; \
+}
+
+/* if (flag==TRUE), then use the contents of randrsl[] to initialize mm[]. */
+void randinit(randctx *ctx, word flag)
+{
+   word i;
+   ub4 a,b,c,d,e,f,g,h;
+   ub4 *m,*r;
+   ctx->randa = ctx->randb = ctx->randc = 0;
+   m=ctx->randmem;
+   r=ctx->randrsl;
+   a=b=c=d=e=f=g=h=0x9e3779b9;  /* the golden ratio */
+
+   for (i=0; i<4; ++i)          /* scramble it */
+   {
+     mix(a,b,c,d,e,f,g,h);
+   }
+
+   if (flag) 
+   {
+     /* initialize using the contents of r[] as the seed */
+     for (i=0; i<RANDSIZ; i+=8)
+     {
+       a+=r[i  ]; b+=r[i+1]; c+=r[i+2]; d+=r[i+3];
+       e+=r[i+4]; f+=r[i+5]; g+=r[i+6]; h+=r[i+7];
+       mix(a,b,c,d,e,f,g,h);
+       m[i  ]=a; m[i+1]=b; m[i+2]=c; m[i+3]=d;
+       m[i+4]=e; m[i+5]=f; m[i+6]=g; m[i+7]=h;
+     }
+     /* do a second pass to make all of the seed affect all of m */
+     for (i=0; i<RANDSIZ; i+=8)
+     {
+       a+=m[i  ]; b+=m[i+1]; c+=m[i+2]; d+=m[i+3];
+       e+=m[i+4]; f+=m[i+5]; g+=m[i+6]; h+=m[i+7];
+       mix(a,b,c,d,e,f,g,h);
+       m[i  ]=a; m[i+1]=b; m[i+2]=c; m[i+3]=d;
+       m[i+4]=e; m[i+5]=f; m[i+6]=g; m[i+7]=h;
+     }
+   }
+   else
+   {
+     for (i=0; i<RANDSIZ; i+=8)
+     {
+       /* fill in mm[] with messy stuff */
+       mix(a,b,c,d,e,f,g,h);
+       m[i  ]=a; m[i+1]=b; m[i+2]=c; m[i+3]=d;
+       m[i+4]=e; m[i+5]=f; m[i+6]=g; m[i+7]=h;
+     }
+   }
+
+   isaac(ctx);            /* fill in the first set of results */
+   ctx->randcnt=RANDSIZ;  /* prepare to use the first set of results */
+}
+
+
+#ifdef NEVER
+int main()
+{
+  ub4 i,j;
+  randctx ctx;
+  ctx.randa=ctx.randb=ctx.randc=(ub4)0;
+  for (i=0; i<256; ++i) ctx.randrsl[i]=(ub4)0;
+  randinit(&ctx, TRUE);
+  for (i=0; i<2; ++i)
+  {
+    isaac(&ctx);
+    for (j=0; j<256; ++j)
+    {
+      printf("%.8lx",ctx.randrsl[j]);
+      if ((j&7)==7) printf("\n");
+    }
+  }
+}
+#endif
diff --git a/src/rt/isaac/standard.h b/src/rt/isaac/standard.h
new file mode 100644
index 00000000..202a5d65
--- /dev/null
+++ b/src/rt/isaac/standard.h
@@ -0,0 +1,57 @@
+/*
+------------------------------------------------------------------------------
+Standard definitions and types, Bob Jenkins
+------------------------------------------------------------------------------
+*/
+#ifndef STANDARD
+# define STANDARD
+# ifndef STDIO
+#  include <stdio.h>
+#  define STDIO
+# endif
+# ifndef STDDEF
+#  include <stddef.h>
+#  define STDDEF
+# endif
+typedef  unsigned long long  ub8;
+#define UB8MAXVAL 0xffffffffffffffffLL
+#define UB8BITS 64
+typedef    signed long long  sb8;
+#define SB8MAXVAL 0x7fffffffffffffffLL
+typedef  unsigned long  int  ub4;   /* unsigned 4-byte quantities */
+#define UB4MAXVAL 0xffffffff
+typedef    signed long  int  sb4;
+#define UB4BITS 32
+#define SB4MAXVAL 0x7fffffff
+typedef  unsigned short int  ub2;
+#define UB2MAXVAL 0xffff
+#define UB2BITS 16
+typedef    signed short int  sb2;
+#define SB2MAXVAL 0x7fff
+typedef  unsigned       char ub1;
+#define UB1MAXVAL 0xff
+#define UB1BITS 8
+typedef    signed       char sb1;   /* signed 1-byte quantities */
+#define SB1MAXVAL 0x7f
+typedef                 int  word;  /* fastest type available */
+
+#define bis(target,mask)  ((target) |=  (mask))
+#define bic(target,mask)  ((target) &= ~(mask))
+#define bit(target,mask)  ((target) &   (mask))
+#ifndef min
+# define min(a,b) (((a)<(b)) ? (a) : (b))
+#endif /* min */
+#ifndef max
+# define max(a,b) (((a)<(b)) ? (b) : (a))
+#endif /* max */
+#ifndef align
+# define align(a) (((ub4)a+(sizeof(void *)-1))&(~(sizeof(void *)-1)))
+#endif /* align */
+#ifndef abs
+# define abs(a)   (((a)>0) ? (a) : -(a))
+#endif
+#define TRUE  1
+#define FALSE 0
+#define SUCCESS 0  /* 1 on VAX */
+
+#endif /* STANDARD */
diff --git a/src/rt/memcheck.h b/src/rt/memcheck.h
new file mode 100644
index 00000000..fc50dabf
--- /dev/null
+++ b/src/rt/memcheck.h
@@ -0,0 +1,309 @@
+
+/*
+   ----------------------------------------------------------------
+
+   Notice that the following BSD-style license applies to this one
+   file (memcheck.h) only.  The rest of Valgrind is licensed under the
+   terms of the GNU General Public License, version 2, unless
+   otherwise indicated.  See the COPYING file in the source
+   distribution for details.
+
+   ----------------------------------------------------------------
+
+   This file is part of MemCheck, a heavyweight Valgrind tool for
+   detecting memory errors.
+
+   Copyright (C) 2000-2009 Julian Seward.  All rights reserved.
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions
+   are met:
+
+   1. Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+   2. The origin of this software must not be misrepresented; you must 
+      not claim that you wrote the original software.  If you use this 
+      software in a product, an acknowledgment in the product 
+      documentation would be appreciated but is not required.
+
+   3. Altered source versions must be plainly marked as such, and must
+      not be misrepresented as being the original software.
+
+   4. The name of the author may not be used to endorse or promote 
+      products derived from this software without specific prior written 
+      permission.
+
+   THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+   OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+   WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+   ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+   DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+   DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+   GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+   WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+   ----------------------------------------------------------------
+
+   Notice that the above BSD-style license applies to this one file
+   (memcheck.h) only.  The entire rest of Valgrind is licensed under
+   the terms of the GNU General Public License, version 2.  See the
+   COPYING file in the source distribution for details.
+
+   ---------------------------------------------------------------- 
+*/
+
+
+#ifndef __MEMCHECK_H
+#define __MEMCHECK_H
+
+
+/* This file is for inclusion into client (your!) code.
+
+   You can use these macros to manipulate and query memory permissions
+   inside your own programs.
+
+   See comment near the top of valgrind.h on how to use them.
+*/
+
+#include "valgrind.h"
+
+/* !! ABIWARNING !! ABIWARNING !! ABIWARNING !! ABIWARNING !! 
+   This enum comprises an ABI exported by Valgrind to programs
+   which use client requests.  DO NOT CHANGE THE ORDER OF THESE
+   ENTRIES, NOR DELETE ANY -- add new ones at the end. */
+typedef
+   enum { 
+      VG_USERREQ__MAKE_MEM_NOACCESS = VG_USERREQ_TOOL_BASE('M','C'),
+      VG_USERREQ__MAKE_MEM_UNDEFINED,
+      VG_USERREQ__MAKE_MEM_DEFINED,
+      VG_USERREQ__DISCARD,
+      VG_USERREQ__CHECK_MEM_IS_ADDRESSABLE,
+      VG_USERREQ__CHECK_MEM_IS_DEFINED,
+      VG_USERREQ__DO_LEAK_CHECK,
+      VG_USERREQ__COUNT_LEAKS,
+
+      VG_USERREQ__GET_VBITS,
+      VG_USERREQ__SET_VBITS,
+
+      VG_USERREQ__CREATE_BLOCK,
+
+      VG_USERREQ__MAKE_MEM_DEFINED_IF_ADDRESSABLE,
+
+      /* Not next to VG_USERREQ__COUNT_LEAKS because it was added later. */
+      VG_USERREQ__COUNT_LEAK_BLOCKS,
+
+      /* This is just for memcheck's internal use - don't use it */
+      _VG_USERREQ__MEMCHECK_RECORD_OVERLAP_ERROR 
+         = VG_USERREQ_TOOL_BASE('M','C') + 256
+   } Vg_MemCheckClientRequest;
+
+
+
+/* Client-code macros to manipulate the state of memory. */
+
+/* Mark memory at _qzz_addr as unaddressable for _qzz_len bytes. */
+#define VALGRIND_MAKE_MEM_NOACCESS(_qzz_addr,_qzz_len)           \
+   (__extension__({unsigned long _qzz_res;                       \
+    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0 /* default return */, \
+                            VG_USERREQ__MAKE_MEM_NOACCESS,       \
+                            _qzz_addr, _qzz_len, 0, 0, 0);       \
+    _qzz_res;                                                    \
+   }))
+      
+/* Similarly, mark memory at _qzz_addr as addressable but undefined
+   for _qzz_len bytes. */
+#define VALGRIND_MAKE_MEM_UNDEFINED(_qzz_addr,_qzz_len)          \
+   (__extension__({unsigned long _qzz_res;                       \
+    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0 /* default return */, \
+                            VG_USERREQ__MAKE_MEM_UNDEFINED,      \
+                            _qzz_addr, _qzz_len, 0, 0, 0);       \
+    _qzz_res;                                                    \
+   }))
+
+/* Similarly, mark memory at _qzz_addr as addressable and defined
+   for _qzz_len bytes. */
+#define VALGRIND_MAKE_MEM_DEFINED(_qzz_addr,_qzz_len)            \
+   (__extension__({unsigned long _qzz_res;                       \
+    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0 /* default return */, \
+                            VG_USERREQ__MAKE_MEM_DEFINED,        \
+                            _qzz_addr, _qzz_len, 0, 0, 0);       \
+    _qzz_res;                                                    \
+   }))
+
+/* Similar to VALGRIND_MAKE_MEM_DEFINED except that addressability is
+   not altered: bytes which are addressable are marked as defined,
+   but those which are not addressable are left unchanged. */
+#define VALGRIND_MAKE_MEM_DEFINED_IF_ADDRESSABLE(_qzz_addr,_qzz_len) \
+   (__extension__({unsigned long _qzz_res;                       \
+    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0 /* default return */, \
+                            VG_USERREQ__MAKE_MEM_DEFINED_IF_ADDRESSABLE, \
+                            _qzz_addr, _qzz_len, 0, 0, 0);       \
+    _qzz_res;                                                    \
+   }))
+
+/* Create a block-description handle.  The description is an ascii
+   string which is included in any messages pertaining to addresses
+   within the specified memory range.  Has no other effect on the
+   properties of the memory range. */
+#define VALGRIND_CREATE_BLOCK(_qzz_addr,_qzz_len, _qzz_desc)	 \
+	(__extension__({unsigned long _qzz_res;			 \
+    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0 /* default return */, \
+                            VG_USERREQ__CREATE_BLOCK,            \
+                            _qzz_addr, _qzz_len, _qzz_desc,      \
+                            0, 0);                               \
+    _qzz_res;							 \
+   }))
+
+/* Discard a block-description-handle. Returns 1 for an
+   invalid handle, 0 for a valid handle. */
+#define VALGRIND_DISCARD(_qzz_blkindex)                          \
+   (__extension__ ({unsigned long _qzz_res;                      \
+    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0 /* default return */, \
+                            VG_USERREQ__DISCARD,                 \
+                            0, _qzz_blkindex, 0, 0, 0);          \
+    _qzz_res;                                                    \
+   }))
+
+
+/* Client-code macros to check the state of memory. */
+
+/* Check that memory at _qzz_addr is addressable for _qzz_len bytes.
+   If suitable addressibility is not established, Valgrind prints an
+   error message and returns the address of the first offending byte.
+   Otherwise it returns zero. */
+#define VALGRIND_CHECK_MEM_IS_ADDRESSABLE(_qzz_addr,_qzz_len)    \
+   (__extension__({unsigned long _qzz_res;                       \
+    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                      \
+                            VG_USERREQ__CHECK_MEM_IS_ADDRESSABLE,\
+                            _qzz_addr, _qzz_len, 0, 0, 0);       \
+    _qzz_res;                                                    \
+   }))
+
+/* Check that memory at _qzz_addr is addressable and defined for
+   _qzz_len bytes.  If suitable addressibility and definedness are not
+   established, Valgrind prints an error message and returns the
+   address of the first offending byte.  Otherwise it returns zero. */
+#define VALGRIND_CHECK_MEM_IS_DEFINED(_qzz_addr,_qzz_len)        \
+   (__extension__({unsigned long _qzz_res;                       \
+    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                      \
+                            VG_USERREQ__CHECK_MEM_IS_DEFINED,    \
+                            _qzz_addr, _qzz_len, 0, 0, 0);       \
+    _qzz_res;                                                    \
+   }))
+
+/* Use this macro to force the definedness and addressibility of an
+   lvalue to be checked.  If suitable addressibility and definedness
+   are not established, Valgrind prints an error message and returns
+   the address of the first offending byte.  Otherwise it returns
+   zero. */
+#define VALGRIND_CHECK_VALUE_IS_DEFINED(__lvalue)                \
+   VALGRIND_CHECK_MEM_IS_DEFINED(                                \
+      (volatile unsigned char *)&(__lvalue),                     \
+                      (unsigned long)(sizeof (__lvalue)))
+
+
+/* Do a full memory leak check (like --leak-check=full) mid-execution. */
+#define VALGRIND_DO_LEAK_CHECK                                   \
+   {unsigned long _qzz_res;                                      \
+    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                      \
+                            VG_USERREQ__DO_LEAK_CHECK,           \
+                            0, 0, 0, 0, 0);                      \
+   }
+
+/* Do a summary memory leak check (like --leak-check=summary) mid-execution. */
+#define VALGRIND_DO_QUICK_LEAK_CHECK				 \
+   {unsigned long _qzz_res;                                      \
+    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                      \
+                            VG_USERREQ__DO_LEAK_CHECK,           \
+                            1, 0, 0, 0, 0);                      \
+   }
+
+/* Return number of leaked, dubious, reachable and suppressed bytes found by
+   all previous leak checks.  They must be lvalues.  */
+#define VALGRIND_COUNT_LEAKS(leaked, dubious, reachable, suppressed)     \
+   /* For safety on 64-bit platforms we assign the results to private
+      unsigned long variables, then assign these to the lvalues the user
+      specified, which works no matter what type 'leaked', 'dubious', etc
+      are.  We also initialise '_qzz_leaked', etc because
+      VG_USERREQ__COUNT_LEAKS doesn't mark the values returned as
+      defined. */                                                        \
+   {unsigned long _qzz_res;                                              \
+    unsigned long _qzz_leaked    = 0, _qzz_dubious    = 0;               \
+    unsigned long _qzz_reachable = 0, _qzz_suppressed = 0;               \
+    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                              \
+                               VG_USERREQ__COUNT_LEAKS,                  \
+                               &_qzz_leaked, &_qzz_dubious,              \
+                               &_qzz_reachable, &_qzz_suppressed, 0);    \
+    leaked     = _qzz_leaked;                                            \
+    dubious    = _qzz_dubious;                                           \
+    reachable  = _qzz_reachable;                                         \
+    suppressed = _qzz_suppressed;                                        \
+   }
+
+/* Return number of leaked, dubious, reachable and suppressed bytes found by
+   all previous leak checks.  They must be lvalues.  */
+#define VALGRIND_COUNT_LEAK_BLOCKS(leaked, dubious, reachable, suppressed) \
+   /* For safety on 64-bit platforms we assign the results to private
+      unsigned long variables, then assign these to the lvalues the user
+      specified, which works no matter what type 'leaked', 'dubious', etc
+      are.  We also initialise '_qzz_leaked', etc because
+      VG_USERREQ__COUNT_LEAKS doesn't mark the values returned as
+      defined. */                                                        \
+   {unsigned long _qzz_res;                                              \
+    unsigned long _qzz_leaked    = 0, _qzz_dubious    = 0;               \
+    unsigned long _qzz_reachable = 0, _qzz_suppressed = 0;               \
+    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                              \
+                               VG_USERREQ__COUNT_LEAK_BLOCKS,            \
+                               &_qzz_leaked, &_qzz_dubious,              \
+                               &_qzz_reachable, &_qzz_suppressed, 0);    \
+    leaked     = _qzz_leaked;                                            \
+    dubious    = _qzz_dubious;                                           \
+    reachable  = _qzz_reachable;                                         \
+    suppressed = _qzz_suppressed;                                        \
+   }
+
+
+/* Get the validity data for addresses [zza..zza+zznbytes-1] and copy it
+   into the provided zzvbits array.  Return values:
+      0   if not running on valgrind
+      1   success
+      2   [previously indicated unaligned arrays;  these are now allowed]
+      3   if any parts of zzsrc/zzvbits are not addressable.
+   The metadata is not copied in cases 0, 2 or 3 so it should be
+   impossible to segfault your system by using this call.
+*/
+#define VALGRIND_GET_VBITS(zza,zzvbits,zznbytes)                 \
+   (__extension__({unsigned long _qzz_res;                       \
+    char* czza     = (char*)zza;                                 \
+    char* czzvbits = (char*)zzvbits;                             \
+    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                      \
+                            VG_USERREQ__GET_VBITS,               \
+                            czza, czzvbits, zznbytes, 0, 0 );    \
+    _qzz_res;                                                    \
+   }))
+
+/* Set the validity data for addresses [zza..zza+zznbytes-1], copying it
+   from the provided zzvbits array.  Return values:
+      0   if not running on valgrind
+      1   success
+      2   [previously indicated unaligned arrays;  these are now allowed]
+      3   if any parts of zza/zzvbits are not addressable.
+   The metadata is not copied in cases 0, 2 or 3 so it should be
+   impossible to segfault your system by using this call.
+*/
+#define VALGRIND_SET_VBITS(zza,zzvbits,zznbytes)                 \
+   (__extension__({unsigned int _qzz_res;                        \
+    char* czza     = (char*)zza;                                 \
+    char* czzvbits = (char*)zzvbits;                             \
+    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                      \
+                            VG_USERREQ__SET_VBITS,               \
+                            czza, czzvbits, zznbytes, 0, 0 );    \
+    _qzz_res;                                                    \
+   }))
+
+#endif
+
diff --git a/src/rt/rust.cpp b/src/rt/rust.cpp
new file mode 100644
index 00000000..8c725bfb
--- /dev/null
+++ b/src/rt/rust.cpp
@@ -0,0 +1,267 @@
+#include "rust_internal.h"
+#include "util/array_list.h"
+
+
+// #define TRACK_ALLOCATIONS
+// For debugging, keeps track of live allocations, so you can find out
+// exactly what leaked.
+
+#ifdef TRACK_ALLOCATIONS
+array_list<void *> allocation_list;
+#endif
+
+rust_srv::rust_srv() :
+    live_allocs(0)
+{
+}
+
+rust_srv::~rust_srv()
+{
+    if (live_allocs != 0) {
+        char msg[128];
+        snprintf(msg, sizeof(msg),
+                 "leaked memory in rust main loop (%" PRIuPTR " objects)",
+                 live_allocs);
+#ifdef TRACK_ALLOCATIONS
+        for (size_t i = 0; i < allocation_list.size(); i++) {
+            if (allocation_list[i] != NULL) {
+                printf("allocation 0x%" PRIxPTR " was not freed\n",
+                        (uintptr_t) allocation_list[i]);
+            }
+        }
+#endif
+        fatal(msg, __FILE__, __LINE__);
+    }
+}
+
+void
+rust_srv::log(char const *str)
+{
+    printf("rt: %s\n", str);
+}
+
+
+
+void *
+rust_srv::malloc(size_t bytes)
+{
+    ++live_allocs;
+    void * val = ::malloc(bytes);
+#ifdef TRACK_ALLOCATIONS
+    allocation_list.append(val);
+#endif
+    return val;
+}
+
+void *
+rust_srv::realloc(void *p, size_t bytes)
+{
+    if (!p) {
+        live_allocs++;
+    }
+    void * val = ::realloc(p, bytes);
+#ifdef TRACK_ALLOCATIONS
+    if (allocation_list.replace(p, val) == NULL) {
+        fatal("not in allocation_list", __FILE__, __LINE__);
+    }
+#endif
+    return val;
+}
+
+void
+rust_srv::free(void *p)
+{
+    if (live_allocs < 1) {
+        fatal("live_allocs < 1", __FILE__, __LINE__);
+    }
+    live_allocs--;
+    ::free(p);
+#ifdef TRACK_ALLOCATIONS
+    if (allocation_list.replace(p, NULL) == NULL) {
+        fatal("not in allocation_list", __FILE__, __LINE__);
+    }
+#endif
+}
+
+void
+rust_srv::fatal(char const *expr, char const *file, size_t line)
+{
+    char buf[1024];
+    snprintf(buf, sizeof(buf),
+             "fatal, '%s' failed, %s:%d",
+             expr, file, (int)line);
+    log(buf);
+    exit(1);
+}
+
+rust_srv *
+rust_srv::clone()
+{
+    return new rust_srv();
+}
+
+
+int
+rust_main_loop(rust_dom *dom)
+{
+    // Make sure someone is watching, to pull us out of infinite loops.
+    rust_timer timer(*dom);
+
+    int rval;
+    rust_task *task;
+
+    dom->log(rust_log::DOM,
+            "running main-loop on domain 0x%" PRIxPTR, dom);
+    dom->logptr("exit-task glue",
+            dom->root_crate->get_exit_task_glue());
+
+    while ((task = dom->sched()) != NULL) {
+        I(dom, task->running());
+
+        dom->log(rust_log::TASK,
+                "activating task 0x%" PRIxPTR ", sp=0x%" PRIxPTR,
+                (uintptr_t)task, task->rust_sp);
+
+        dom->interrupt_flag = 0;
+
+        dom->activate(task);
+
+        dom->log(rust_log::TASK,
+                 "returned from task 0x%" PRIxPTR
+                 " in state '%s', sp=0x%" PRIxPTR,
+                 (uintptr_t)task,
+                 dom->state_vec_name(task->state),
+                 task->rust_sp);
+
+        I(dom, task->rust_sp >= (uintptr_t) &task->stk->data[0]);
+        I(dom, task->rust_sp < task->stk->limit);
+
+        dom->reap_dead_tasks();
+    }
+
+    dom->log(rust_log::DOM, "finished main-loop (dom.rval = %d)", dom->rval);
+    rval = dom->rval;
+
+    return rval;
+}
+
+
+struct
+command_line_args
+{
+    rust_dom &dom;
+    int argc;
+    char **argv;
+
+    // vec[str] passed to rust_task::start.
+    rust_vec *args;
+
+    command_line_args(rust_dom &dom,
+                      int sys_argc,
+                      char **sys_argv)
+        : dom(dom),
+          argc(sys_argc),
+          argv(sys_argv),
+          args(NULL)
+    {
+#if defined(__WIN32__)
+        LPCWSTR cmdline = GetCommandLineW();
+        LPWSTR *wargv = CommandLineToArgvW(cmdline, &argc);
+        dom.win32_require("CommandLineToArgvW", argv != NULL);
+        argv = (char **) dom.malloc(sizeof(char*) * argc);
+        for (int i = 0; i < argc; ++i) {
+            int n_chars = WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1,
+                                              NULL, 0, NULL, NULL);
+            dom.win32_require("WideCharToMultiByte(0)", n_chars != 0);
+            argv[i] = (char *) dom.malloc(n_chars);
+            n_chars = WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1,
+                                          argv[i], n_chars, NULL, NULL);
+            dom.win32_require("WideCharToMultiByte(1)", n_chars != 0);
+        }
+        LocalFree(wargv);
+#endif
+        size_t vec_fill = sizeof(rust_str *) * argc;
+        size_t vec_alloc = next_power_of_two(sizeof(rust_vec) + vec_fill);
+        void *mem = dom.malloc(vec_alloc);
+        args = new (mem) rust_vec(&dom, vec_alloc, 0, NULL);
+        rust_str **strs = (rust_str**) &args->data[0];
+        for (int i = 0; i < argc; ++i) {
+            size_t str_fill = strlen(argv[i]) + 1;
+            size_t str_alloc = next_power_of_two(sizeof(rust_str) + str_fill);
+            mem = dom.malloc(str_alloc);
+            strs[i] = new (mem) rust_str(&dom, str_alloc, str_fill,
+                                         (uint8_t const *)argv[i]);
+        }
+        args->fill = vec_fill;
+        // If the caller has a declared args array, they may drop; but
+        // we don't know if they have such an array. So we pin the args
+        // array here to ensure it survives to program-shutdown.
+        args->ref();
+    }
+
+    ~command_line_args() {
+        if (args) {
+            // Drop the args we've had pinned here.
+            rust_str **strs = (rust_str**) &args->data[0];
+            for (int i = 0; i < argc; ++i)
+                dom.free(strs[i]);
+            dom.free(args);
+        }
+
+#ifdef __WIN32__
+        for (int i = 0; i < argc; ++i) {
+            dom.free(argv[i]);
+        }
+        dom.free(argv);
+#endif
+    }
+};
+
+
+extern "C" CDECL int
+rust_start(uintptr_t main_fn, rust_crate const *crate, int argc, char **argv)
+{
+    int ret;
+    {
+        rust_srv srv;
+        rust_dom dom(&srv, crate);
+        command_line_args args(dom, argc, argv);
+
+        dom.log(rust_log::DOM, "startup: %d args", args.argc);
+        for (int i = 0; i < args.argc; ++i)
+            dom.log(rust_log::DOM,
+                    "startup: arg[%d] = '%s'", i, args.argv[i]);
+
+        if (dom._log.is_tracing(rust_log::DWARF)) {
+            rust_crate_reader rdr(&dom, crate);
+        }
+
+        uintptr_t main_args[3] = { 0, 0, (uintptr_t)args.args };
+
+        dom.root_task->start(crate->get_exit_task_glue(),
+                             main_fn,
+                             (uintptr_t)&main_args,
+                             sizeof(main_args));
+
+        ret = rust_main_loop(&dom);
+    }
+
+#if !defined(__WIN32__)
+    // Don't take down the process if the main thread exits without an
+    // error.
+    if (!ret)
+        pthread_exit(NULL);
+#endif
+    return ret;
+}
+
+//
+// Local Variables:
+// mode: C++
+// fill-column: 78;
+// indent-tabs-mode: nil
+// c-basic-offset: 4
+// buffer-file-coding-system: utf-8-unix
+// compile-command: "make -k -C .. 2>&1 | sed -e 's/\\/x\\//x:\\//g'";
+// End:
+//
diff --git a/src/rt/rust.h b/src/rt/rust.h
new file mode 100644
index 00000000..135a1799
--- /dev/null
+++ b/src/rt/rust.h
@@ -0,0 +1,49 @@
+#ifndef RUST_H
+#define RUST_H
+
+/*
+ * Include this file after you've defined the ISO C9x stdint
+ * types (size_t, uint8_t, uintptr_t, etc.)
+ */
+
+#ifdef __i386__
+// 'cdecl' ABI only means anything on i386
+#ifdef __WIN32__
+#define CDECL __cdecl
+#else
+#define CDECL __attribute__((cdecl))
+#endif
+#else
+#define CDECL
+#endif
+
+struct rust_srv {
+    size_t live_allocs;
+
+    virtual void log(char const *);
+    virtual void fatal(char const *, char const *, size_t);
+    virtual void *malloc(size_t);
+    virtual void *realloc(void *, size_t);
+    virtual void free(void *);
+    virtual rust_srv *clone();
+
+    rust_srv();
+    virtual ~rust_srv();
+};
+
+inline void *operator new(size_t size, rust_srv *srv)
+{
+    return srv->malloc(size);
+}
+
+/*
+ * Local Variables:
+ * fill-column: 78;
+ * indent-tabs-mode: nil
+ * c-basic-offset: 4
+ * buffer-file-coding-system: utf-8-unix
+ * compile-command: "make -k -C .. 2>&1 | sed -e 's/\\/x\\//x:\\//g'";
+ * End:
+ */
+
+#endif /* RUST_H */
diff --git a/src/rt/rust_builtin.cpp b/src/rt/rust_builtin.cpp
new file mode 100644
index 00000000..71aa644b
--- /dev/null
+++ b/src/rt/rust_builtin.cpp
@@ -0,0 +1,129 @@
+
+#include "rust_internal.h"
+
+/* Native builtins. */
+extern "C" CDECL rust_str*
+str_alloc(rust_task *task, size_t n_bytes)
+{
+    rust_dom *dom = task->dom;
+    size_t alloc = next_power_of_two(sizeof(rust_str) + n_bytes);
+    void *mem = dom->malloc(alloc);
+    if (!mem) {
+        task->fail(2);
+        return NULL;
+    }
+    rust_str *st = new (mem) rust_str(dom, alloc, 1, (uint8_t const *)"");
+    return st;
+}
+
+extern "C" CDECL rust_str*
+last_os_error(rust_task *task) {
+    rust_dom *dom = task->dom;
+    dom->log(rust_log::TASK, "last_os_error()");
+
+#if defined(__WIN32__)
+    LPTSTR buf;
+    DWORD err = GetLastError();
+    DWORD res = FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER |
+                              FORMAT_MESSAGE_FROM_SYSTEM |
+                              FORMAT_MESSAGE_IGNORE_INSERTS,
+                              NULL, err,
+                              MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+                              (LPTSTR) &buf, 0, NULL);
+    if (!res) {
+        task->fail(1);
+        return NULL;
+    }
+#elif defined(_GNU_SOURCE)
+    char cbuf[1024];
+    char *buf = strerror_r(errno, cbuf, sizeof(cbuf));
+    if (!buf) {
+        task->fail(1);
+        return NULL;
+    }
+#else
+    char buf[1024];
+    int err = strerror_r(errno, buf, sizeof(buf));
+    if (err) {
+        task->fail(1);
+        return NULL;
+    }
+#endif
+    size_t fill = strlen(buf) + 1;
+    size_t alloc = next_power_of_two(sizeof(rust_str) + fill);
+    void *mem = dom->malloc(alloc);
+    if (!mem) {
+        task->fail(1);
+        return NULL;
+    }
+    rust_str *st = new (mem) rust_str(dom, alloc, fill, (const uint8_t *)buf);
+
+#ifdef __WIN32__
+    LocalFree((HLOCAL)buf);
+#endif
+    return st;
+}
+
+extern "C" CDECL size_t
+size_of(rust_task *task, type_desc *t) {
+  return t->size;
+}
+
+extern "C" CDECL size_t
+align_of(rust_task *task, type_desc *t) {
+  return t->align;
+}
+
+extern "C" CDECL size_t
+refcount(rust_task *task, type_desc *t, size_t *v) {
+    // Passed-in value has refcount 1 too high
+    // because it was ref'ed while making the call.
+    return (*v) - 1;
+}
+
+extern "C" CDECL rust_vec*
+vec_alloc(rust_task *task, type_desc *t, size_t n_elts)
+{
+    rust_dom *dom = task->dom;
+    dom->log(rust_log::MEM,
+            "vec_alloc %" PRIdPTR " elements of size %" PRIdPTR,
+             n_elts, t->size);
+    size_t fill = n_elts * t->size;
+    size_t alloc = next_power_of_two(sizeof(rust_vec) + fill);
+    void *mem = dom->malloc(alloc);
+    if (!mem) {
+        task->fail(3);
+        return NULL;
+    }
+    rust_vec *vec = new (mem) rust_vec(dom, alloc, 0, NULL);
+    return vec;
+}
+
+extern "C" CDECL char const *
+str_buf(rust_task *task, rust_str *s)
+{
+    return (char const *)&s->data[0];
+}
+
+extern "C" CDECL void *
+vec_buf(rust_task *task, type_desc *ty, rust_vec *v)
+{
+    return (void *)&v->data[0];
+}
+
+extern "C" CDECL size_t
+vec_len(rust_task *task, type_desc *ty, rust_vec *v)
+{
+    return v->fill;
+}
+
+//
+// Local Variables:
+// mode: C++
+// fill-column: 78;
+// indent-tabs-mode: nil
+// c-basic-offset: 4
+// buffer-file-coding-system: utf-8-unix
+// compile-command: "make -k -C .. 2>&1 | sed -e 's/\\/x\\//x:\\//g'";
+// End:
+//
diff --git a/src/rt/rust_chan.cpp b/src/rt/rust_chan.cpp
new file mode 100644
index 00000000..38f93a7d
--- /dev/null
+++ b/src/rt/rust_chan.cpp
@@ -0,0 +1,34 @@
+
+#include "rust_internal.h"
+#include "rust_chan.h"
+
+rust_chan::rust_chan(rust_task *task, rust_port *port) :
+    task(task),
+    port(port),
+    buffer(task->dom, port->unit_sz),
+    token(this)
+{
+    if (port)
+        port->chans.push(this);
+}
+
+rust_chan::~rust_chan()
+{
+    if (port) {
+        if (token.pending())
+            token.withdraw();
+        port->chans.swapdel(this);
+    }
+}
+
+void
+rust_chan::disassociate()
+{
+    I(task->dom, port);
+
+    if (token.pending())
+        token.withdraw();
+
+    // Delete reference to the port/
+    port = NULL;
+}
diff --git a/src/rt/rust_chan.h b/src/rt/rust_chan.h
new file mode 100644
index 00000000..a56ba0ca
--- /dev/null
+++ b/src/rt/rust_chan.h
@@ -0,0 +1,22 @@
+
+#ifndef RUST_CHAN_H
+#define RUST_CHAN_H
+
+class rust_chan : public rc_base<rust_chan>, public task_owned<rust_chan> {
+public:
+    rust_chan(rust_task *task, rust_port *port);
+    ~rust_chan();
+
+    rust_task *task;
+    rust_port *port;
+    circ_buf buffer;
+    size_t idx;           // Index into port->chans.
+
+    // Token belonging to this chan, it will be placed into a port's
+    // writers vector if we have something to send to the port.
+    rust_token token;
+
+    void disassociate();
+};
+
+#endif /* RUST_CHAN_H */
diff --git a/src/rt/rust_comm.cpp b/src/rt/rust_comm.cpp
new file mode 100644
index 00000000..58b9ef4c
--- /dev/null
+++ b/src/rt/rust_comm.cpp
@@ -0,0 +1,199 @@
+
+#include "rust_internal.h"
+
+template class ptr_vec<rust_token>;
+template class ptr_vec<rust_alarm>;
+template class ptr_vec<rust_chan>;
+
+rust_alarm::rust_alarm(rust_task *receiver) :
+    receiver(receiver)
+{
+}
+
+
+// Circular buffers.
+
+circ_buf::circ_buf(rust_dom *dom, size_t unit_sz) :
+    dom(dom),
+    alloc(INIT_CIRC_BUF_UNITS * unit_sz),
+    unit_sz(unit_sz),
+    next(0),
+    unread(0),
+    data((uint8_t *)dom->calloc(alloc))
+{
+    I(dom, unit_sz);
+    dom->log(rust_log::MEM|rust_log::COMM,
+             "new circ_buf(alloc=%d, unread=%d) -> circ_buf=0x%" PRIxPTR,
+             alloc, unread, this);
+    I(dom, data);
+}
+
+circ_buf::~circ_buf()
+{
+    dom->log(rust_log::MEM|rust_log::COMM,
+             "~circ_buf 0x%" PRIxPTR,
+             this);
+    I(dom, data);
+    // I(dom, unread == 0);
+    dom->free(data);
+}
+
+void
+circ_buf::transfer(void *dst)
+{
+    size_t i;
+    uint8_t *d = (uint8_t *)dst;
+    I(dom, dst);
+    for (i = 0; i < unread; i += unit_sz)
+        memcpy(&d[i], &data[next + i % alloc], unit_sz);
+}
+
+void
+circ_buf::push(void *src)
+{
+    size_t i;
+    void *tmp;
+
+    I(dom, src);
+    I(dom, unread <= alloc);
+
+    /* Grow if necessary. */
+    if (unread == alloc) {
+        I(dom, alloc <= MAX_CIRC_BUF_SIZE);
+        tmp = dom->malloc(alloc << 1);
+        transfer(tmp);
+        alloc <<= 1;
+        dom->free(data);
+        data = (uint8_t *)tmp;
+    }
+
+    dom->log(rust_log::MEM|rust_log::COMM,
+             "circ buf push, unread=%d, alloc=%d, unit_sz=%d",
+             unread, alloc, unit_sz);
+
+    I(dom, unread < alloc);
+    I(dom, unread + unit_sz <= alloc);
+
+    i = (next + unread) % alloc;
+    memcpy(&data[i], src, unit_sz);
+
+    dom->log(rust_log::MEM|rust_log::COMM, "pushed data at index %d", i);
+    unread += unit_sz;
+}
+
+void
+circ_buf::shift(void *dst)
+{
+    size_t i;
+    void *tmp;
+
+    I(dom, dst);
+    I(dom, unit_sz > 0);
+    I(dom, unread >= unit_sz);
+    I(dom, unread <= alloc);
+    I(dom, data);
+    i = next;
+    memcpy(dst, &data[i], unit_sz);
+    dom->log(rust_log::MEM|rust_log::COMM, "shifted data from index %d", i);
+    unread -= unit_sz;
+    next += unit_sz;
+    I(dom, next <= alloc);
+    if (next == alloc)
+        next = 0;
+
+    /* Shrink if necessary. */
+    if (alloc >= INIT_CIRC_BUF_UNITS * unit_sz &&
+        unread <= alloc / 4) {
+        tmp = dom->malloc(alloc / 2);
+        transfer(tmp);
+        alloc >>= 1;
+        dom->free(data);
+        data = (uint8_t *)tmp;
+    }
+}
+
+
+// Ports.
+
+rust_port::rust_port(rust_task *task, size_t unit_sz) :
+    task(task),
+    unit_sz(unit_sz),
+    writers(task->dom),
+    chans(task->dom)
+{
+    rust_dom *dom = task->dom;
+    dom->log(rust_log::MEM|rust_log::COMM,
+             "new rust_port(task=0x%" PRIxPTR ", unit_sz=%d) -> port=0x%"
+             PRIxPTR, (uintptr_t)task, unit_sz, (uintptr_t)this);
+}
+
+rust_port::~rust_port()
+{
+    rust_dom *dom = task->dom;
+    dom->log(rust_log::COMM|rust_log::MEM,
+             "~rust_port 0x%" PRIxPTR,
+             (uintptr_t)this);
+    while (chans.length() > 0)
+        chans.pop()->disassociate();
+}
+
+
+// Tokens.
+
+rust_token::rust_token(rust_chan *chan) :
+    chan(chan),
+    idx(0),
+    submitted(false)
+{
+}
+
+rust_token::~rust_token()
+{
+}
+
+bool
+rust_token::pending() const
+{
+    return submitted;
+}
+
+void
+rust_token::submit()
+{
+    rust_port *port = chan->port;
+    rust_dom *dom = chan->task->dom;
+
+    I(dom, port);
+    I(dom, !submitted);
+
+    port->writers.push(this);
+    submitted = true;
+}
+
+void
+rust_token::withdraw()
+{
+    rust_task *task = chan->task;
+    rust_port *port = chan->port;
+    rust_dom *dom = task->dom;
+
+    I(dom, port);
+    I(dom, submitted);
+
+    if (task->blocked())
+        task->wakeup(this); // must be blocked on us (or dead)
+    port->writers.swapdel(this);
+    submitted = false;
+}
+
+
+//
+// Local Variables:
+// mode: C++
+// fill-column: 78;
+// indent-tabs-mode: nil
+// c-basic-offset: 4
+// buffer-file-coding-system: utf-8-unix
+// compile-command: "make -k -C .. 2>&1 | sed -e 's/\\/x\\//x:\\//g'";
+// End:
+//
diff --git a/src/rt/rust_crate.cpp b/src/rt/rust_crate.cpp
new file mode 100644
index 00000000..d609ac64
--- /dev/null
+++ b/src/rt/rust_crate.cpp
@@ -0,0 +1,63 @@
+
+#include "rust_internal.h"
+
+uintptr_t
+rust_crate::get_image_base() const {
+  return ((uintptr_t)this + image_base_off);
+}
+
+ptrdiff_t
+rust_crate::get_relocation_diff() const {
+  return ((uintptr_t)this - self_addr);
+}
+
+activate_glue_ty
+rust_crate::get_activate_glue() const {
+  return (activate_glue_ty) ((uintptr_t)this + activate_glue_off);
+}
+
+uintptr_t
+rust_crate::get_exit_task_glue() const {
+  return ((uintptr_t)this + exit_task_glue_off);
+}
+
+uintptr_t
+rust_crate::get_unwind_glue() const {
+  return ((uintptr_t)this + unwind_glue_off);
+}
+
+uintptr_t
+rust_crate::get_yield_glue() const {
+  return ((uintptr_t)this + yield_glue_off);
+}
+
+rust_crate::mem_area::mem_area(rust_dom *dom, uintptr_t pos, size_t sz)
+  : dom(dom),
+    base(pos),
+    lim(pos + sz)
+{
+  dom->log(rust_log::MEM, "new mem_area [0x%" PRIxPTR ",0x%" PRIxPTR "]",
+           base, lim);
+}
+
+rust_crate::mem_area
+rust_crate::get_debug_info(rust_dom *dom) const {
+  return mem_area(dom, ((uintptr_t)this + debug_info_off),
+                  debug_info_sz);
+}
+
+rust_crate::mem_area
+rust_crate::get_debug_abbrev(rust_dom *dom) const {
+  return mem_area(dom, ((uintptr_t)this + debug_abbrev_off),
+                  debug_abbrev_sz);
+}
+
+//
+// Local Variables:
+// mode: C++
+// fill-column: 78;
+// indent-tabs-mode: nil
+// c-basic-offset: 4
+// buffer-file-coding-system: utf-8-unix
+// compile-command: "make -k -C .. 2>&1 | sed -e 's/\\/x\\//x:\\//g'";
+// End:
diff --git a/src/rt/rust_crate_cache.cpp b/src/rt/rust_crate_cache.cpp
new file mode 100644
index 00000000..fa10b91b
--- /dev/null
+++ b/src/rt/rust_crate_cache.cpp
@@ -0,0 +1,306 @@
+
+#include "rust_internal.h"
+
+rust_crate_cache::lib::lib(rust_dom *dom, char const *name)
+    : handle(0),
+      dom(dom)
+{
+#if defined(__WIN32__)
+    handle = (uintptr_t)LoadLibrary(_T(name));
+#else
+    handle = (uintptr_t)dlopen(name, RTLD_LOCAL|RTLD_LAZY);
+#endif
+    dom->log(rust_log::CACHE, "loaded library '%s' as 0x%"  PRIxPTR,
+             name, handle);
+}
+
+rust_crate_cache::lib::~lib() {
+    dom->log(rust_log::CACHE, "~rust_crate_cache::lib(0x%" PRIxPTR ")",
+             handle);
+    if (handle) {
+#if defined(__WIN32__)
+        FreeLibrary((HMODULE)handle);
+#else
+        dlclose((void*)handle);
+#endif
+    }
+}
+
+uintptr_t
+rust_crate_cache::lib::get_handle() {
+    return handle;
+}
+
+
+
+rust_crate_cache::c_sym::c_sym(rust_dom *dom, lib *library, char const *name)
+    : val(0),
+      library(library),
+      dom(dom)
+{
+    library->ref();
+    uintptr_t handle = library->get_handle();
+    if (handle) {
+#if defined(__WIN32__)
+        val = (uintptr_t)GetProcAddress((HMODULE)handle, _T(name));
+#else
+        val = (uintptr_t)dlsym((void*)handle, name);
+#endif
+        dom->log(rust_log::CACHE, "resolved symbol '%s' to 0x%"  PRIxPTR,
+                 name, val);
+    } else {
+        dom->log(rust_log::CACHE, "unresolved symbol '%s', null lib handle",
+                 name);
+    }
+}
+
+rust_crate_cache::c_sym::~c_sym() {
+    dom->log(rust_log::CACHE,
+            "~rust_crate_cache::c_sym(0x%" PRIxPTR ")", val);
+    library->deref();
+}
+
+uintptr_t
+rust_crate_cache::c_sym::get_val() {
+    return val;
+}
+
+
+
+rust_crate_cache::rust_sym::rust_sym(rust_dom *dom,
+                                     rust_crate const *curr_crate,
+                                     c_sym *crate_sym,
+                                     char const **path)
+    : val(0),
+      crate_sym(crate_sym),
+      dom(dom)
+{
+    crate_sym->ref();
+    typedef rust_crate_reader::die die;
+    rust_crate const *crate = (rust_crate*)crate_sym->get_val();
+    if (!crate) {
+        dom->log(rust_log::CACHE,
+                 "failed to resolve symbol, null crate symbol");
+        return;
+    }
+    rust_crate_reader rdr(dom, crate);
+    bool found_root = false;
+    bool found_leaf = false;
+    for (die d = rdr.dies.first_die();
+         !(found_root || d.is_null());
+         d = d.next_sibling()) {
+
+        die t1 = d;
+        die t2 = d;
+        for (char const **c = crate_rel(curr_crate, path);
+             (*c
+              && !t1.is_null()
+              && t1.find_child_by_name(crate_rel(curr_crate, *c), t2));
+             ++c, t1=t2) {
+            dom->log(rust_log::DWARF|rust_log::CACHE,
+                    "matched die <0x%"  PRIxPTR
+                    ">, child '%s' = die<0x%" PRIxPTR ">",
+                    t1.off, crate_rel(curr_crate, *c), t2.off);
+            found_root = found_root || true;
+            if (!*(c+1) && t2.find_num_attr(DW_AT_low_pc, val)) {
+                dom->log(rust_log::DWARF|rust_log::CACHE,
+                         "found relative address: 0x%"  PRIxPTR, val);
+                dom->log(rust_log::DWARF|rust_log::CACHE,
+                         "plus image-base 0x%"  PRIxPTR,
+                         crate->get_image_base());
+                val += crate->get_image_base();
+                found_leaf = true;
+                break;
+            }
+        }
+        if (found_root || found_leaf)
+            break;
+    }
+    if (found_leaf) {
+        dom->log(rust_log::CACHE, "resolved symbol to 0x%"  PRIxPTR, val);
+    } else {
+        dom->log(rust_log::CACHE, "failed to resolve symbol");
+    }
+}
+
+rust_crate_cache::rust_sym::~rust_sym() {
+    dom->log(rust_log::CACHE,
+             "~rust_crate_cache::rust_sym(0x%" PRIxPTR ")", val);
+    crate_sym->deref();
+}
+
+uintptr_t
+rust_crate_cache::rust_sym::get_val() {
+    return val;
+}
+
+
+
+rust_crate_cache::lib *
+rust_crate_cache::get_lib(size_t n, char const *name)
+{
+    I(dom, n < crate->n_libs);
+    lib *library = libs[n];
+    if (!library) {
+        library = new (dom) lib(dom, name);
+        libs[n] = library;
+    }
+    return library;
+}
+
+rust_crate_cache::c_sym *
+rust_crate_cache::get_c_sym(size_t n, lib *library, char const *name)
+{
+    I(dom, n < crate->n_c_syms);
+    c_sym *sym = c_syms[n];
+    dom->log(rust_log::CACHE, "cached C symbol %s = 0x%" PRIxPTR, name, sym);
+    if (!sym) {
+        sym = new (dom) c_sym(dom, library, name);
+        c_syms[n] = sym;
+    }
+    return sym;
+}
+
+rust_crate_cache::rust_sym *
+rust_crate_cache::get_rust_sym(size_t n,
+                               rust_dom *dom,
+                               rust_crate const *curr_crate,
+                               c_sym *crate_sym,
+                               char const **path)
+{
+    I(dom, n < crate->n_rust_syms);
+    rust_sym *sym = rust_syms[n];
+    if (!sym) {
+        sym = new (dom) rust_sym(dom, curr_crate, crate_sym, path);
+        rust_syms[n] = sym;
+    }
+    return sym;
+}
+
+static inline void
+adjust_disp(uintptr_t &disp, const void *oldp, const void *newp)
+{
+    if (disp) {
+        disp += (uintptr_t)oldp;
+        disp -= (uintptr_t)newp;
+    }
+}
+
+type_desc *
+rust_crate_cache::get_type_desc(size_t size,
+                                size_t align,
+                                size_t n_descs,
+                                type_desc const **descs)
+{
+    I(dom, n_descs > 1);
+    type_desc *td = NULL;
+    size_t keysz = n_descs * sizeof(type_desc*);
+    HASH_FIND(hh, this->type_descs, descs, keysz, td);
+    if (td) {
+        dom->log(rust_log::CACHE, "rust_crate_cache::get_type_desc hit");
+        return td;
+    }
+    dom->log(rust_log::CACHE, "rust_crate_cache::get_type_desc miss");
+    td = (type_desc*) dom->malloc(sizeof(type_desc) + keysz);
+    if (!td)
+        return NULL;
+    // By convention, desc 0 is the root descriptor.
+    // but we ignore the size and alignment of it and use the
+    // passed-in, computed values.
+    memcpy(td, descs[0], sizeof(type_desc));
+    td->first_param = &td->descs[1];
+    td->size = size;
+    td->align = align;
+    for (size_t i = 0; i < n_descs; ++i) {
+        dom->log(rust_log::CACHE,
+                 "rust_crate_cache::descs[%" PRIdPTR "] = 0x%" PRIxPTR,
+                 i, descs[i]);
+        td->descs[i] = descs[i];
+    }
+    adjust_disp(td->copy_glue_off, descs[0], td);
+    adjust_disp(td->drop_glue_off, descs[0], td);
+    adjust_disp(td->free_glue_off, descs[0], td);
+    adjust_disp(td->mark_glue_off, descs[0], td);
+    adjust_disp(td->obj_drop_glue_off, descs[0], td);
+    HASH_ADD(hh, this->type_descs, descs, keysz, td);
+    return td;
+}
+
+rust_crate_cache::rust_crate_cache(rust_dom *dom,
+                                   rust_crate const *crate)
+    : rust_syms((rust_sym**)
+                dom->calloc(sizeof(rust_sym*) * crate->n_rust_syms)),
+      c_syms((c_sym**) dom->calloc(sizeof(c_sym*) * crate->n_c_syms)),
+      libs((lib**) dom->calloc(sizeof(lib*) * crate->n_libs)),
+      type_descs(NULL),
+      crate(crate),
+      dom(dom),
+      idx(0)
+{
+    I(dom, rust_syms);
+    I(dom, c_syms);
+    I(dom, libs);
+}
+
+void
+rust_crate_cache::flush() {
+    dom->log(rust_log::CACHE, "rust_crate_cache::flush()");
+    for (size_t i = 0; i < crate->n_rust_syms; ++i) {
+        rust_sym *s = rust_syms[i];
+        if (s) {
+            dom->log(rust_log::CACHE,
+                     "rust_crate_cache::flush() deref rust_sym %"
+                     PRIdPTR " (rc=%" PRIdPTR ")", i, s->refcnt);
+            s->deref();
+        }
+        rust_syms[i] = NULL;
+    }
+
+    for (size_t i = 0; i < crate->n_c_syms; ++i) {
+        c_sym *s = c_syms[i];
+        if (s) {
+            dom->log(rust_log::CACHE,
+                     "rust_crate_cache::flush() deref c_sym %"
+                     PRIdPTR " (rc=%" PRIdPTR ")", i, s->refcnt);
+            s->deref();
+        }
+        c_syms[i] = NULL;
+    }
+
+    for (size_t i = 0; i < crate->n_libs; ++i) {
+        lib *l = libs[i];
+        if (l) {
+            dom->log(rust_log::CACHE, "rust_crate_cache::flush() deref lib %"
+                     PRIdPTR " (rc=%" PRIdPTR ")", i, l->refcnt);
+            l->deref();
+        }
+        libs[i] = NULL;
+    }
+
+    while (type_descs) {
+        type_desc *d = type_descs;
+        HASH_DEL(type_descs, d);
+        dom->log(rust_log::MEM,
+                 "rust_crate_cache::flush() tydesc %" PRIxPTR, d);
+        dom->free(d);
+    }
+}
+
+rust_crate_cache::~rust_crate_cache()
+{
+    flush();
+    dom->free(rust_syms);
+    dom->free(c_syms);
+    dom->free(libs);
+}
+
+//
+// Local Variables:
+// mode: C++
+// fill-column: 78;
+// indent-tabs-mode: nil
+// c-basic-offset: 4
+// buffer-file-coding-system: utf-8-unix
+// compile-command: "make -k -C .. 2>&1 | sed -e 's/\\/x\\//x:\\//g'";
+// End:
+//
diff --git a/src/rt/rust_crate_reader.cpp b/src/rt/rust_crate_reader.cpp
new file mode 100644
index 00000000..3c36729f
--- /dev/null
+++ b/src/rt/rust_crate_reader.cpp
@@ -0,0 +1,578 @@
+
+#include "rust_internal.h"
+
+bool
+rust_crate_reader::mem_reader::is_ok()
+{
+  return ok;
+}
+
+bool
+rust_crate_reader::mem_reader::at_end()
+{
+  return pos == mem.lim;
+}
+
+void
+rust_crate_reader::mem_reader::fail()
+{
+  ok = false;
+}
+
+void
+rust_crate_reader::mem_reader::reset()
+{
+  pos = mem.base;
+  ok = true;
+}
+
+rust_crate_reader::mem_reader::mem_reader(rust_crate::mem_area &m)
+  : mem(m),
+    ok(true),
+    pos(m.base)
+{}
+
+size_t
+rust_crate_reader::mem_reader::tell_abs()
+{
+  return pos;
+}
+
+size_t
+rust_crate_reader::mem_reader::tell_off()
+{
+  return pos - mem.base;
+}
+
+void
+rust_crate_reader::mem_reader::seek_abs(uintptr_t p)
+{
+  if (!ok || p < mem.base || p >= mem.lim)
+    ok = false;
+  else
+    pos = p;
+}
+
+void
+rust_crate_reader::mem_reader::seek_off(uintptr_t p)
+{
+  seek_abs(p + mem.base);
+}
+
+
+bool
+rust_crate_reader::mem_reader::adv_zstr(size_t sz)
+{
+  sz = 0;
+  while (ok) {
+    char c;
+    get(c);
+    ++sz;
+    if (c == '\0')
+      return true;
+  }
+  return false;
+}
+
+bool
+rust_crate_reader::mem_reader::get_zstr(char const *&c, size_t &sz)
+{
+  if (!ok)
+    return false;
+  c = (char const *)(pos);
+  return adv_zstr(sz);
+}
+
+void
+rust_crate_reader::mem_reader::adv(size_t amt)
+{
+  if (pos < mem.base
+      || pos >= mem.lim
+      || pos + amt > mem.lim)
+    ok = false;
+  if (!ok)
+    return;
+  // mem.dom->log(rust_log::MEM, "adv %d bytes", amt);
+  pos += amt;
+  ok &= !at_end();
+  I(mem.dom, at_end() || (mem.base <= pos && pos < mem.lim));
+}
+
+
+rust_crate_reader::abbrev::abbrev(rust_dom *dom,
+                                  uintptr_t body_off,
+                                  size_t body_sz,
+                                  uintptr_t tag,
+                                  uint8_t has_children) :
+  dom(dom),
+  body_off(body_off),
+  tag(tag),
+  has_children(has_children),
+  idx(0)
+{}
+
+
+rust_crate_reader::abbrev_reader::abbrev_reader
+  (rust_crate::mem_area &abbrev_mem)
+  : mem_reader(abbrev_mem),
+    abbrevs(abbrev_mem.dom)
+{
+  rust_dom *dom = mem.dom;
+  while (is_ok()) {
+
+    // dom->log(rust_log::DWARF, "reading new abbrev at 0x%" PRIxPTR,
+    //          tell_off());
+
+    uintptr_t idx, tag;
+    uint8_t has_children;
+    get_uleb(idx);
+    get_uleb(tag);
+    get(has_children);
+
+    uintptr_t attr, form;
+    size_t body_off = tell_off();
+    while (is_ok() && step_attr_form_pair(attr, form));
+
+    // dom->log(rust_log::DWARF,
+    //         "finished scanning attr/form pairs, pos=0x%"
+    //         PRIxPTR ", lim=0x%" PRIxPTR ", is_ok=%d, at_end=%d",
+    //        pos, mem.lim, is_ok(), at_end());
+
+    if (is_ok() || at_end()) {
+      dom->log(rust_log::DWARF, "read abbrev: %" PRIdPTR, idx);
+      I(dom, idx = abbrevs.length() + 1);
+      abbrevs.push(new (dom) abbrev(dom, body_off,
+                                    tell_off() - body_off,
+                                    tag, has_children));
+    }
+  }
+}
+
+rust_crate_reader::abbrev *
+rust_crate_reader::abbrev_reader::get_abbrev(size_t i) {
+  i -= 1;
+  if (i < abbrevs.length())
+    return abbrevs[i];
+  return NULL;
+}
+
+bool
+rust_crate_reader::abbrev_reader::step_attr_form_pair(uintptr_t &attr,
+                                                      uintptr_t &form)
+{
+  attr = 0;
+  form = 0;
+  // mem.dom->log(rust_log::DWARF, "reading attr/form pair at 0x%" PRIxPTR,
+  //              tell_off());
+  get_uleb(attr);
+  get_uleb(form);
+  // mem.dom->log(rust_log::DWARF, "attr 0x%" PRIxPTR ", form 0x%" PRIxPTR,
+  //              attr, form);
+  return ! (attr == 0 && form == 0);
+}
+rust_crate_reader::abbrev_reader::~abbrev_reader() {
+  while (abbrevs.length()) {
+    delete abbrevs.pop();
+  }
+}
+
+
+bool
+rust_crate_reader::attr::is_numeric() const
+{
+  switch (form) {
+  case DW_FORM_ref_addr:
+  case DW_FORM_addr:
+  case DW_FORM_data4:
+  case DW_FORM_data1:
+  case DW_FORM_flag:
+    return true;
+  default:
+    break;
+  }
+  return false;
+}
+
+bool
+rust_crate_reader::attr::is_string() const
+{
+  return form == DW_FORM_string;
+}
+
+size_t
+rust_crate_reader::attr::get_ssz(rust_dom *dom) const
+{
+  I(dom, is_string());
+  return val.str.sz;
+}
+
+char const *
+rust_crate_reader::attr::get_str(rust_dom *dom) const
+{
+  I(dom, is_string());
+  return val.str.s;
+}
+
+uintptr_t
+rust_crate_reader::attr::get_num(rust_dom *dom) const
+{
+  I(dom, is_numeric());
+  return val.num;
+}
+
+bool
+rust_crate_reader::attr::is_unknown() const {
+  return !(is_numeric() || is_string());
+}
+
+rust_crate_reader::rdr_sess::rdr_sess(die_reader *rdr) : rdr(rdr)
+{
+  I(rdr->mem.dom, !rdr->in_use);
+  rdr->in_use = true;
+}
+
+rust_crate_reader::rdr_sess::~rdr_sess()
+{
+  rdr->in_use = false;
+}
+
+rust_crate_reader::die::die(die_reader *rdr, uintptr_t off)
+  : rdr(rdr),
+    off(off),
+    using_rdr(false)
+{
+  rust_dom *dom = rdr->mem.dom;
+  rdr_sess use(rdr);
+
+  rdr->reset();
+  rdr->seek_off(off);
+  if (!rdr->is_ok()) {
+    ab = NULL;
+    return;
+  }
+  size_t ab_idx;
+  rdr->get_uleb(ab_idx);
+  if (!ab_idx) {
+    ab = NULL;
+    dom->log(rust_log::DWARF, "DIE <0x%" PRIxPTR "> (null)", off);
+  } else {
+    ab = rdr->abbrevs.get_abbrev(ab_idx);
+    dom->log(rust_log::DWARF, "DIE <0x%" PRIxPTR "> abbrev 0x%"
+             PRIxPTR, off, ab_idx);
+    dom->log(rust_log::DWARF, "  tag 0x%x, has children: %d",
+             ab->tag, ab->has_children);
+  }
+}
+
+bool
+rust_crate_reader::die::is_null() const
+{
+  return ab == NULL;
+}
+
+bool
+rust_crate_reader::die::has_children() const
+{
+  return (!is_null()) && ab->has_children;
+}
+
+dw_tag
+rust_crate_reader::die::tag() const
+{
+  if (is_null())
+    return (dw_tag) (-1);
+  return (dw_tag) ab->tag;
+}
+
+bool
+rust_crate_reader::die::start_attrs() const
+{
+  if (is_null())
+    return false;
+  rdr->reset();
+  rdr->seek_off(off + 1);
+  rdr->abbrevs.reset();
+  rdr->abbrevs.seek_off(ab->body_off);
+  return rdr->is_ok();
+}
+
+bool
+rust_crate_reader::die::step_attr(attr &a) const
+{
+  uintptr_t ai, fi;
+  if (rdr->abbrevs.step_attr_form_pair(ai, fi) && rdr->is_ok()) {
+    a.at = (dw_at)ai;
+    a.form = (dw_form)fi;
+
+    uint32_t u32;
+    uint8_t u8;
+
+    switch (a.form) {
+    case DW_FORM_string:
+      return rdr->get_zstr(a.val.str.s, a.val.str.sz);
+      break;
+
+    case DW_FORM_ref_addr:
+      I(rdr->mem.dom, sizeof(uintptr_t) == 4);
+    case DW_FORM_addr:
+    case DW_FORM_data4:
+      rdr->get(u32);
+      a.val.num = (uintptr_t)u32;
+      return rdr->is_ok() || rdr->at_end();
+      break;
+
+    case DW_FORM_data1:
+    case DW_FORM_flag:
+      rdr->get(u8);
+      a.val.num = u8;
+      return rdr->is_ok() || rdr->at_end();
+      break;
+
+    case DW_FORM_block1:
+      rdr->get(u8);
+      rdr->adv(u8);
+      return rdr->is_ok() || rdr->at_end();
+      break;
+
+    default:
+      rdr->mem.dom->log(rust_log::DWARF, "  unknown dwarf form: 0x%"
+                        PRIxPTR, a.form);
+      rdr->fail();
+      break;
+    }
+  }
+  return false;
+}
+
+bool
+rust_crate_reader::die::find_str_attr(dw_at at, char const *&c)
+{
+  rdr_sess use(rdr);
+  if (is_null())
+    return false;
+  if (start_attrs()) {
+    attr a;
+    while (step_attr(a)) {
+      if (a.at == at && a.is_string()) {
+        c = a.get_str(rdr->mem.dom);
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
+bool
+rust_crate_reader::die::find_num_attr(dw_at at, uintptr_t &n)
+{
+  rdr_sess use(rdr);
+  if (is_null())
+    return false;
+  if (start_attrs()) {
+    attr a;
+    while (step_attr(a)) {
+      if (a.at == at && a.is_numeric()) {
+        n = a.get_num(rdr->mem.dom);
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
+bool
+rust_crate_reader::die::is_transparent()
+{
+  // "semantically transparent" DIEs are those with
+  // children that serve to structure the tree but have
+  // tags that don't reflect anything in the rust-module
+  // name hierarchy.
+  switch (tag()) {
+  case DW_TAG_compile_unit:
+  case DW_TAG_lexical_block:
+    return (has_children());
+  default:
+    break;
+  }
+  return false;
+}
+
+bool
+rust_crate_reader::die::find_child_by_name(char const *c,
+                                                       die &child,
+                                                       bool exact)
+{
+  rust_dom *dom = rdr->mem.dom;
+  I(dom, has_children());
+  I(dom, !is_null());
+
+  for (die ch = next(); !ch.is_null(); ch = ch.next_sibling()) {
+    char const *ac;
+    if (!exact && ch.is_transparent()) {
+      if (ch.find_child_by_name(c, child, exact)) {
+        return true;
+      }
+    }
+    else if (ch.find_str_attr(DW_AT_name, ac)) {
+      if (strcmp(ac, c) == 0) {
+        child = ch;
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
+bool
+rust_crate_reader::die::find_child_by_tag(dw_tag tag, die &child)
+{
+  rust_dom *dom = rdr->mem.dom;
+  I(dom, has_children());
+  I(dom, !is_null());
+
+  for (child = next(); !child.is_null();
+       child = child.next_sibling()) {
+    if (child.tag() == tag)
+      return true;
+  }
+  return false;
+}
+
+rust_crate_reader::die
+rust_crate_reader::die::next() const
+{
+  rust_dom *dom = rdr->mem.dom;
+
+  if (is_null()) {
+    rdr->seek_off(off + 1);
+    return die(rdr, rdr->tell_off());
+  }
+
+  {
+    rdr_sess use(rdr);
+    if (start_attrs()) {
+      attr a;
+      while (step_attr(a)) {
+        I(dom, !(a.is_numeric() && a.is_string()));
+        if (a.is_numeric())
+          dom->log(rust_log::DWARF, "  attr num: 0x%"
+                   PRIxPTR, a.get_num(dom));
+        else if (a.is_string())
+          dom->log(rust_log::DWARF, "  attr str: %s",
+                   a.get_str(dom));
+        else
+          dom->log(rust_log::DWARF, "  attr ??:");
+      }
+    }
+  }
+  return die(rdr, rdr->tell_off());
+}
+
+rust_crate_reader::die
+rust_crate_reader::die::next_sibling() const
+{
+  // FIXME: use DW_AT_sibling, when present.
+  if (has_children()) {
+    // rdr->mem.dom->log(rust_log::DWARF, "+++ children of die 0x%"
+    //                   PRIxPTR, off);
+    die child = next();
+    while (!child.is_null())
+      child = child.next_sibling();
+    // rdr->mem.dom->log(rust_log::DWARF, "--- children of die 0x%"
+    //                   PRIxPTR, off);
+    return child.next();
+  } else {
+    return next();
+  }
+}
+
+
+rust_crate_reader::die
+rust_crate_reader::die_reader::first_die()
+{
+  reset();
+  seek_off(cu_base
+           + sizeof(dwarf_vers)
+           + sizeof(cu_abbrev_off)
+           + sizeof(sizeof_addr));
+  return die(this, tell_off());
+}
+
+void
+rust_crate_reader::die_reader::dump()
+{
+  rust_dom *dom = mem.dom;
+  die d = first_die();
+  while (!d.is_null())
+    d = d.next_sibling();
+  I(dom, d.is_null());
+  I(dom, d.off == mem.lim - mem.base);
+}
+
+
+rust_crate_reader::die_reader::die_reader(rust_crate::mem_area &die_mem,
+                              abbrev_reader &abbrevs)
+  : mem_reader(die_mem),
+    abbrevs(abbrevs),
+    cu_unit_length(0),
+    cu_base(0),
+    dwarf_vers(0),
+    cu_abbrev_off(0),
+    sizeof_addr(0),
+    in_use(false)
+{
+  rust_dom *dom = mem.dom;
+
+  rdr_sess use(this);
+
+  get(cu_unit_length);
+  cu_base = tell_off();
+
+  get(dwarf_vers);
+  get(cu_abbrev_off);
+  get(sizeof_addr);
+
+  if (is_ok()) {
+    dom->log(rust_log::DWARF, "new root CU at 0x%" PRIxPTR, die_mem.base);
+    dom->log(rust_log::DWARF, "CU unit length: %" PRId32, cu_unit_length);
+    dom->log(rust_log::DWARF, "dwarf version: %" PRId16, dwarf_vers);
+    dom->log(rust_log::DWARF, "CU abbrev off: %" PRId32, cu_abbrev_off);
+    dom->log(rust_log::DWARF, "size of address: %" PRId8, sizeof_addr);
+    I(dom, sizeof_addr == sizeof(uintptr_t));
+    I(dom, dwarf_vers >= 2);
+    I(dom, cu_base + cu_unit_length == die_mem.lim - die_mem.base);
+  } else {
+    dom->log(rust_log::DWARF, "failed to read root CU header");
+  }
+}
+
+rust_crate_reader::die_reader::~die_reader() {
+}
+
+
+rust_crate_reader::rust_crate_reader(rust_dom *dom,
+                                     rust_crate const *crate)
+  : dom(dom),
+    crate(crate),
+    abbrev_mem(crate->get_debug_abbrev(dom)),
+    abbrevs(abbrev_mem),
+    die_mem(crate->get_debug_info(dom)),
+    dies(die_mem, abbrevs)
+{
+  dom->log(rust_log::MEM, "crate_reader on crate: 0x%" PRIxPTR, this);
+  dom->log(rust_log::MEM, "debug_abbrev: 0x%" PRIxPTR, abbrev_mem.base);
+  dom->log(rust_log::MEM, "debug_info: 0x%" PRIxPTR, die_mem.base);
+  // For now, perform diagnostics only.
+  dies.dump();
+}
+
+
+//
+// Local Variables:
+// mode: C++
+// fill-column: 78;
+// indent-tabs-mode: nil
+// c-basic-offset: 4
+// buffer-file-coding-system: utf-8-unix
+// compile-command: "make -k -C .. 2>&1 | sed -e 's/\\/x\\//x:\\//g'";
+// End:
diff --git a/src/rt/rust_dom.cpp b/src/rt/rust_dom.cpp
new file mode 100644
index 00000000..3b5e23b2
--- /dev/null
+++ b/src/rt/rust_dom.cpp
@@ -0,0 +1,271 @@
+
+#include <stdarg.h>
+#include "rust_internal.h"
+
+template class ptr_vec<rust_task>;
+
+rust_dom::rust_dom(rust_srv *srv, rust_crate const *root_crate) :
+    interrupt_flag(0),
+    root_crate(root_crate),
+    _log(srv, this),
+    srv(srv),
+    running_tasks(this),
+    blocked_tasks(this),
+    dead_tasks(this),
+    caches(this),
+    root_task(NULL),
+    curr_task(NULL),
+    rval(0)
+{
+    logptr("new dom", (uintptr_t)this);
+    memset(&rctx, 0, sizeof(rctx));
+
+#ifdef __WIN32__
+    {
+        HCRYPTPROV hProv;
+        win32_require
+            (_T("CryptAcquireContext"),
+             CryptAcquireContext(&hProv, NULL, NULL, PROV_RSA_FULL,
+                                 CRYPT_VERIFYCONTEXT|CRYPT_SILENT));
+        win32_require
+            (_T("CryptGenRandom"),
+             CryptGenRandom(hProv, sizeof(rctx.randrsl),
+                            (BYTE*)(&rctx.randrsl)));
+        win32_require
+            (_T("CryptReleaseContext"),
+             CryptReleaseContext(hProv, 0));
+    }
+#else
+    int fd = open("/dev/urandom", O_RDONLY);
+    I(this, fd > 0);
+    I(this, read(fd, (void*) &rctx.randrsl, sizeof(rctx.randrsl))
+      == sizeof(rctx.randrsl));
+    I(this, close(fd) == 0);
+    pthread_attr_init(&attr);
+    pthread_attr_setstacksize(&attr, 1024 * 1024);
+    pthread_attr_setdetachstate(&attr, true);
+#endif
+    randinit(&rctx, 1);
+
+    root_task = new (this) rust_task(this, NULL);
+}
+
+static void
+del_all_tasks(rust_dom *dom, ptr_vec<rust_task> *v) {
+    I(dom, v);
+    while (v->length()) {
+        dom->log(rust_log::TASK, "deleting task %" PRIdPTR, v->length() - 1);
+        delete v->pop();
+    }
+}
+
+rust_dom::~rust_dom() {
+    log(rust_log::TASK, "deleting all running tasks");
+    del_all_tasks(this, &running_tasks);
+    log(rust_log::TASK, "deleting all blocked tasks");
+    del_all_tasks(this, &blocked_tasks);
+    log(rust_log::TASK, "deleting all dead tasks");
+    del_all_tasks(this, &dead_tasks);
+#ifndef __WIN32__
+    pthread_attr_destroy(&attr);
+#endif
+    while (caches.length())
+        delete caches.pop();
+}
+
+void
+rust_dom::activate(rust_task *task) {
+    curr_task = task;
+    root_crate->get_activate_glue()(task);
+    curr_task = NULL;
+}
+
+void
+rust_dom::log(uint32_t type_bits, char const *fmt, ...) {
+    char buf[256];
+    if (_log.is_tracing(type_bits)) {
+        va_list args;
+        va_start(args, fmt);
+        vsnprintf(buf, sizeof(buf), fmt, args);
+        _log.trace_ln(type_bits, buf);
+        va_end(args);
+    }
+}
+
+rust_log &
+rust_dom::get_log() {
+    return _log;
+}
+
+void
+rust_dom::logptr(char const *msg, uintptr_t ptrval) {
+    log(rust_log::MEM, "%s 0x%" PRIxPTR, msg, ptrval);
+}
+
+template<typename T> void
+rust_dom::logptr(char const *msg, T* ptrval) {
+    log(rust_log::MEM, "%s 0x%" PRIxPTR, msg, (uintptr_t)ptrval);
+}
+
+
+void
+rust_dom::fail() {
+    log(rust_log::DOM, "domain 0x%" PRIxPTR " root task failed", this);
+    I(this, rval == 0);
+    rval = 1;
+}
+
+void *
+rust_dom::malloc(size_t sz) {
+    void *p = srv->malloc(sz);
+    I(this, p);
+    log(rust_log::MEM, "rust_dom::malloc(%d) -> 0x%" PRIxPTR,
+        sz, p);
+    return p;
+}
+
+void *
+rust_dom::calloc(size_t sz) {
+    void *p = this->malloc(sz);
+    memset(p, 0, sz);
+    return p;
+}
+
+void *
+rust_dom::realloc(void *p, size_t sz) {
+    void *p1 = srv->realloc(p, sz);
+    I(this, p1);
+    log(rust_log::MEM, "rust_dom::realloc(0x%" PRIxPTR ", %d) -> 0x%" PRIxPTR,
+        p, sz, p1);
+    return p1;
+}
+
+void
+rust_dom::free(void *p) {
+    log(rust_log::MEM, "rust_dom::free(0x%" PRIxPTR ")", p);
+    I(this, p);
+    srv->free(p);
+}
+
+#ifdef __WIN32__
+void
+rust_dom::win32_require(LPCTSTR fn, BOOL ok) {
+    if (!ok) {
+        LPTSTR buf;
+        DWORD err = GetLastError();
+        FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER |
+                      FORMAT_MESSAGE_FROM_SYSTEM |
+                      FORMAT_MESSAGE_IGNORE_INSERTS,
+                      NULL, err,
+                      MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+                      (LPTSTR) &buf, 0, NULL );
+        log(rust_log::ERR, "%s failed with error %ld: %s", fn, err, buf);
+        LocalFree((HLOCAL)buf);
+        I(this, ok);
+    }
+}
+#endif
+
+size_t
+rust_dom::n_live_tasks()
+{
+    return running_tasks.length() + blocked_tasks.length();
+}
+
+void
+rust_dom::add_task_to_state_vec(ptr_vec<rust_task> *v, rust_task *task)
+{
+    log(rust_log::MEM|rust_log::TASK,
+        "adding task 0x%" PRIxPTR " in state '%s' to vec 0x%" PRIxPTR,
+        (uintptr_t)task, state_vec_name(v), (uintptr_t)v);
+    v->push(task);
+}
+
+
+void
+rust_dom::remove_task_from_state_vec(ptr_vec<rust_task> *v, rust_task *task)
+{
+    log(rust_log::MEM|rust_log::TASK,
+        "removing task 0x%" PRIxPTR " in state '%s' from vec 0x%" PRIxPTR,
+        (uintptr_t)task, state_vec_name(v), (uintptr_t)v);
+    I(this, (*v)[task->idx] == task);
+    v->swapdel(task);
+}
+
+const char *
+rust_dom::state_vec_name(ptr_vec<rust_task> *v)
+{
+    if (v == &running_tasks)
+        return "running";
+    if (v == &blocked_tasks)
+        return "blocked";
+    I(this, v == &dead_tasks);
+    return "dead";
+}
+
+void
+rust_dom::reap_dead_tasks()
+{
+    for (size_t i = 0; i < dead_tasks.length(); ) {
+        rust_task *t = dead_tasks[i];
+        if (t == root_task || t->refcnt == 0) {
+            I(this, !t->waiting_tasks.length());
+            dead_tasks.swapdel(t);
+            log(rust_log::TASK,
+                "deleting unreferenced dead task 0x%" PRIxPTR, t);
+            delete t;
+            continue;
+        }
+        ++i;
+    }
+}
+
+rust_task *
+rust_dom::sched()
+{
+    I(this, this);
+    // FIXME: in the face of failing tasks, this is not always right.
+    // I(this, n_live_tasks() > 0);
+    if (running_tasks.length() > 0) {
+        size_t i = rand(&rctx);
+        i %= running_tasks.length();
+        return (rust_task *)running_tasks[i];
+    }
+    log(rust_log::DOM|rust_log::TASK,
+        "no schedulable tasks");
+    return NULL;
+}
+
+rust_crate_cache *
+rust_dom::get_cache(rust_crate const *crate) {
+    log(rust_log::CACHE,
+        "looking for crate-cache for crate 0x%" PRIxPTR, crate);
+    rust_crate_cache *cache = NULL;
+    for (size_t i = 0; i < caches.length(); ++i) {
+        rust_crate_cache *c = caches[i];
+        if (c->crate == crate) {
+            cache = c;
+            break;
+        }
+    }
+    if (!cache) {
+        log(rust_log::CACHE,
+            "making new crate-cache for crate 0x%" PRIxPTR, crate);
+        cache = new (this) rust_crate_cache(this, crate);
+        caches.push(cache);
+    }
+    cache->ref();
+    return cache;
+}
+
+
+//
+// Local Variables:
+// mode: C++
+// fill-column: 70;
+// indent-tabs-mode: nil
+// c-basic-offset: 4
+// buffer-file-coding-system: utf-8-unix
+// compile-command: "make -k -C .. 2>&1 | sed -e 's/\\/x\\//x:\\//g'";
+// End:
+//
diff --git a/src/rt/rust_dwarf.h b/src/rt/rust_dwarf.h
new file mode 100644
index 00000000..8eff3b8c
--- /dev/null
+++ b/src/rt/rust_dwarf.h
@@ -0,0 +1,198 @@
+#ifndef RUST_DWARF_H
+#define RUST_DWARF_H
+
+enum
+dw_form
+  {
+    DW_FORM_addr = 0x01,
+    DW_FORM_block2 = 0x03,
+    DW_FORM_block4 = 0x04,
+    DW_FORM_data2 = 0x05,
+    DW_FORM_data4 = 0x06,
+    DW_FORM_data8 = 0x07,
+    DW_FORM_string = 0x08,
+    DW_FORM_block = 0x09,
+    DW_FORM_block1 = 0x0a,
+    DW_FORM_data1 = 0x0b,
+    DW_FORM_flag = 0x0c,
+    DW_FORM_sdata = 0x0d,
+    DW_FORM_strp = 0x0e,
+    DW_FORM_udata = 0x0f,
+    DW_FORM_ref_addr = 0x10,
+    DW_FORM_ref1 = 0x11,
+    DW_FORM_ref2 = 0x12,
+    DW_FORM_ref4 = 0x13,
+    DW_FORM_ref8 = 0x14,
+    DW_FORM_ref_udata = 0x15,
+    DW_FORM_indirect = 0x16
+  };
+
+enum
+dw_at
+  {
+    DW_AT_sibling = 0x01,
+    DW_AT_location = 0x02,
+    DW_AT_name = 0x03,
+    DW_AT_ordering = 0x09,
+    DW_AT_byte_size = 0x0b,
+    DW_AT_bit_offset = 0x0c,
+    DW_AT_bit_size = 0x0d,
+    DW_AT_stmt_list = 0x10,
+    DW_AT_low_pc = 0x11,
+    DW_AT_high_pc = 0x12,
+    DW_AT_language = 0x13,
+    DW_AT_discr = 0x15,
+    DW_AT_discr_value = 0x16,
+    DW_AT_visibility = 0x17,
+    DW_AT_import = 0x18,
+    DW_AT_string_length = 0x19,
+    DW_AT_common_reference = 0x1a,
+    DW_AT_comp_dir = 0x1b,
+    DW_AT_const_value = 0x1c,
+    DW_AT_containing_type = 0x1d,
+    DW_AT_default_value = 0x1e,
+    DW_AT_inline = 0x20,
+    DW_AT_is_optional = 0x21,
+    DW_AT_lower_bound = 0x22,
+    DW_AT_producer = 0x25,
+    DW_AT_prototyped = 0x27,
+    DW_AT_return_addr = 0x2a,
+    DW_AT_start_scope = 0x2c,
+    DW_AT_bit_stride = 0x2e,
+    DW_AT_upper_bound = 0x2f,
+    DW_AT_abstract_origin = 0x31,
+    DW_AT_accessibility = 0x32,
+    DW_AT_address_class = 0x33,
+    DW_AT_artificial = 0x34,
+    DW_AT_base_types = 0x35,
+    DW_AT_calling_convention = 0x36,
+    DW_AT_count = 0x37,
+    DW_AT_data_member_location = 0x38,
+    DW_AT_decl_column = 0x39,
+    DW_AT_decl_file = 0x3a,
+    DW_AT_decl_line = 0x3b,
+    DW_AT_declaration = 0x3c,
+    DW_AT_discr_list = 0x3d,
+    DW_AT_encoding = 0x3e,
+    DW_AT_external = 0x3f,
+    DW_AT_frame_base = 0x40,
+    DW_AT_friend = 0x41,
+    DW_AT_identifier_case = 0x42,
+    DW_AT_macro_info = 0x43,
+    DW_AT_namelist_item = 0x44,
+    DW_AT_priority = 0x45,
+    DW_AT_segment = 0x46,
+    DW_AT_specification = 0x47,
+    DW_AT_static_link = 0x48,
+    DW_AT_type = 0x49,
+    DW_AT_use_location = 0x4a,
+    DW_AT_variable_parameter = 0x4b,
+    DW_AT_virtuality = 0x4c,
+    DW_AT_vtable_elem_location = 0x4d,
+    DW_AT_allocated = 0x4e,
+    DW_AT_associated = 0x4f,
+    DW_AT_data_location = 0x50,
+    DW_AT_byte_stride = 0x51,
+    DW_AT_entry_pc = 0x52,
+    DW_AT_use_UTF8 = 0x53,
+    DW_AT_extension = 0x54,
+    DW_AT_ranges = 0x55,
+    DW_AT_trampoline = 0x56,
+    DW_AT_call_column = 0x57,
+    DW_AT_call_file = 0x58,
+    DW_AT_call_line = 0x59,
+    DW_AT_description = 0x5a,
+    DW_AT_binary_scale = 0x5b,
+    DW_AT_decimal_scale = 0x5c,
+    DW_AT_small = 0x5d,
+    DW_AT_decimal_sign = 0x5e,
+    DW_AT_digit_count = 0x5f,
+    DW_AT_picture_string = 0x60,
+    DW_AT_mutable = 0x61,
+    DW_AT_threads_scaled = 0x62,
+    DW_AT_explicit = 0x63,
+    DW_AT_object_pointer = 0x64,
+    DW_AT_endianity = 0x65,
+    DW_AT_elemental = 0x66,
+    DW_AT_pure = 0x67,
+    DW_AT_recursive = 0x68,
+    DW_AT_lo_user = 0x2000,
+    DW_AT_hi_user = 0x3fff
+};
+
+enum
+dw_tag
+  {
+    DW_TAG_array_type = 0x01,
+    DW_TAG_class_type = 0x02,
+    DW_TAG_entry_point = 0x03,
+    DW_TAG_enumeration_type = 0x04,
+    DW_TAG_formal_parameter = 0x05,
+    DW_TAG_imported_declaration = 0x08,
+    DW_TAG_label = 0x0a,
+    DW_TAG_lexical_block = 0x0b,
+    DW_TAG_member = 0x0d,
+    DW_TAG_pointer_type = 0x0f,
+    DW_TAG_reference_type = 0x10,
+    DW_TAG_compile_unit = 0x11,
+    DW_TAG_string_type = 0x12,
+    DW_TAG_structure_type = 0x13,
+    DW_TAG_subroutine_type = 0x15,
+    DW_TAG_typedef = 0x16,
+    DW_TAG_union_type = 0x17,
+    DW_TAG_unspecified_parameters = 0x18,
+    DW_TAG_variant = 0x19,
+    DW_TAG_common_block = 0x1a,
+    DW_TAG_common_inclusion = 0x1b,
+    DW_TAG_inheritance = 0x1c,
+    DW_TAG_inlined_subroutine = 0x1d,
+    DW_TAG_module = 0x1e,
+    DW_TAG_ptr_to_member_type = 0x1f,
+    DW_TAG_set_type = 0x20,
+    DW_TAG_subrange_type = 0x21,
+    DW_TAG_with_stmt = 0x22,
+    DW_TAG_access_declaration = 0x23,
+    DW_TAG_base_type = 0x24,
+    DW_TAG_catch_block = 0x25,
+    DW_TAG_const_type = 0x26,
+    DW_TAG_constant = 0x27,
+    DW_TAG_enumerator = 0x28,
+    DW_TAG_file_type = 0x29,
+    DW_TAG_friend = 0x2a,
+    DW_TAG_namelist = 0x2b,
+    DW_TAG_namelist_item = 0x2c,
+    DW_TAG_packed_type = 0x2d,
+    DW_TAG_subprogram = 0x2e,
+    DW_TAG_template_type_parameter = 0x2f,
+    DW_TAG_template_value_parameter = 0x30,
+    DW_TAG_thrown_type = 0x31,
+    DW_TAG_try_block = 0x32,
+    DW_TAG_variant_part = 0x33,
+    DW_TAG_variable = 0x34,
+    DW_TAG_volatile_type = 0x35,
+    DW_TAG_dwarf_procedure = 0x36,
+    DW_TAG_restrict_type = 0x37,
+    DW_TAG_interface_type = 0x38,
+    DW_TAG_namespace = 0x39,
+    DW_TAG_imported_module = 0x3a,
+    DW_TAG_unspecified_type = 0x3b,
+    DW_TAG_partial_unit = 0x3c,
+    DW_TAG_imported_unit = 0x3d,
+    DW_TAG_condition = 0x3f,
+    DW_TAG_shared_type = 0x40,
+    DW_TAG_lo_user = 0x4080,
+    DW_TAG_hi_user = 0xffff,
+  };
+
+//
+// Local Variables:
+// mode: C++
+// fill-column: 78;
+// indent-tabs-mode: nil
+// c-basic-offset: 4
+// buffer-file-coding-system: utf-8-unix
+// compile-command: "make -k -C .. 2>&1 | sed -e 's/\\/x\\//x:\\//g'";
+// End:
+//
+
+#endif
diff --git a/src/rt/rust_internal.h b/src/rt/rust_internal.h
new file mode 100644
index 00000000..c393b210
--- /dev/null
+++ b/src/rt/rust_internal.h
@@ -0,0 +1,730 @@
+#ifndef RUST_INTERNAL_H
+#define RUST_INTERNAL_H
+
+#define __STDC_LIMIT_MACROS 1
+#define __STDC_CONSTANT_MACROS 1
+#define __STDC_FORMAT_MACROS 1
+
+#include <stdlib.h>
+#include <stdint.h>
+#include <inttypes.h>
+
+#include <stdio.h>
+#include <string.h>
+
+#include "rust.h"
+
+#include "rand.h"
+#include "rust_log.h"
+#include "uthash.h"
+
+#if defined(__WIN32__)
+extern "C" {
+#include <windows.h>
+#include <tchar.h>
+#include <wincrypt.h>
+}
+#elif defined(__GNUC__)
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <dlfcn.h>
+#include <pthread.h>
+#include <errno.h>
+#else
+#error "Platform not supported."
+#endif
+
+#ifndef __i386__
+#error "Target CPU not supported."
+#endif
+
+#define I(dom, e) ((e) ? (void)0 :                              \
+                   (dom)->srv->fatal(#e, __FILE__, __LINE__))
+
+struct rust_task;
+struct rust_port;
+class rust_chan;
+struct rust_token;
+struct rust_dom;
+class rust_crate;
+class rust_crate_cache;
+class lockfree_queue;
+
+struct stk_seg;
+struct type_desc;
+struct frame_glue_fns;
+
+// This drives our preemption scheme.
+
+static size_t const TIME_SLICE_IN_MS = 10;
+
+// Every reference counted object should derive from this base class.
+
+template <typename T>
+struct
+rc_base
+{
+    size_t refcnt;
+
+    void ref() {
+        ++refcnt;
+    }
+
+    void deref() {
+        if (--refcnt == 0) {
+            delete (T*)this;
+        }
+    }
+
+  rc_base();
+  ~rc_base();
+};
+
+template <typename T>
+struct
+dom_owned
+{
+    void operator delete(void *ptr) {
+        ((T *)ptr)->dom->free(ptr);
+    }
+};
+
+template <typename T>
+struct
+task_owned
+{
+    void operator delete(void *ptr) {
+        ((T *)ptr)->task->dom->free(ptr);
+    }
+};
+
+
+// Helper class used regularly elsewhere.
+
+template <typename T>
+class
+ptr_vec : public dom_owned<ptr_vec<T> >
+{
+    static const size_t INIT_SIZE = 8;
+
+    rust_dom *dom;
+    size_t alloc;
+    size_t fill;
+    T **data;
+
+public:
+    ptr_vec(rust_dom *dom);
+    ~ptr_vec();
+
+    size_t length() {
+        return fill;
+    }
+
+    T *& operator[](size_t offset);
+    void push(T *p);
+    T *pop();
+    void trim(size_t fill);
+    void swapdel(T* p);
+};
+
+struct
+rust_dom
+{
+    // Fields known to the compiler:
+    uintptr_t interrupt_flag;
+
+    // Fields known only by the runtime:
+
+    // NB: the root crate must remain in memory until the root of the
+    // tree of domains exits. All domains within this tree have a
+    // copy of this root_crate value and use it for finding utility
+    // glue.
+    rust_crate const *root_crate;
+    rust_log _log;
+    rust_srv *srv;
+    // uint32_t logbits;
+    ptr_vec<rust_task> running_tasks;
+    ptr_vec<rust_task> blocked_tasks;
+    ptr_vec<rust_task> dead_tasks;
+    ptr_vec<rust_crate_cache> caches;
+    randctx rctx;
+    rust_task *root_task;
+    rust_task *curr_task;
+    int rval;
+    lockfree_queue *incoming; // incoming messages from other threads
+
+#ifndef __WIN32__
+    pthread_attr_t attr;
+#endif
+
+    rust_dom(rust_srv *srv, rust_crate const *root_crate);
+    ~rust_dom();
+
+    void activate(rust_task *task);
+    void log(uint32_t logbit, char const *fmt, ...);
+    rust_log & get_log();
+    void logptr(char const *msg, uintptr_t ptrval);
+    template<typename T>
+    void logptr(char const *msg, T* ptrval);
+    void fail();
+    void *malloc(size_t sz);
+    void *calloc(size_t sz);
+    void *realloc(void *data, size_t sz);
+    void free(void *p);
+
+#ifdef __WIN32__
+    void win32_require(LPCTSTR fn, BOOL ok);
+#endif
+
+    rust_crate_cache *get_cache(rust_crate const *crate);
+    size_t n_live_tasks();
+    void add_task_to_state_vec(ptr_vec<rust_task> *v, rust_task *task);
+    void remove_task_from_state_vec(ptr_vec<rust_task> *v, rust_task *task);
+    const char *state_vec_name(ptr_vec<rust_task> *v);
+
+    void reap_dead_tasks();
+    rust_task *sched();
+};
+
+inline void *operator new(size_t sz, void *mem) {
+    return mem;
+}
+
+inline void *operator new(size_t sz, rust_dom *dom) {
+    return dom->malloc(sz);
+}
+
+inline void *operator new[](size_t sz, rust_dom *dom) {
+    return dom->malloc(sz);
+}
+
+inline void *operator new(size_t sz, rust_dom &dom) {
+    return dom.malloc(sz);
+}
+
+inline void *operator new[](size_t sz, rust_dom &dom) {
+    return dom.malloc(sz);
+}
+
+struct
+rust_timer
+{
+    // FIXME: This will probably eventually need replacement
+    // with something more sophisticated and integrated with
+    // an IO event-handling library, when we have such a thing.
+    // For now it's just the most basic "thread that can interrupt
+    // its associated domain-thread" device, so that we have
+    // *some* form of task-preemption.
+    rust_dom &dom;
+    uintptr_t exit_flag;
+
+#if defined(__WIN32__)
+    HANDLE thread;
+#else
+    pthread_attr_t attr;
+    pthread_t thread;
+#endif
+
+    rust_timer(rust_dom &dom);
+    ~rust_timer();
+};
+
+#include "rust_util.h"
+
+// Crates.
+
+template<typename T> T*
+crate_rel(rust_crate const *crate, T *t) {
+    return (T*)(((uintptr_t)crate) + ((ptrdiff_t)t));
+}
+
+template<typename T> T const*
+crate_rel(rust_crate const *crate, T const *t) {
+    return (T const*)(((uintptr_t)crate) + ((ptrdiff_t)t));
+}
+
+typedef void CDECL (*activate_glue_ty)(rust_task *);
+
+class
+rust_crate
+{
+    // The following fields are emitted by the compiler for the static
+    // rust_crate object inside each compiled crate.
+
+    ptrdiff_t image_base_off;     // (Loaded image base) - this.
+    uintptr_t self_addr;          // Un-relocated addres of 'this'.
+
+    ptrdiff_t debug_abbrev_off;   // Offset from this to .debug_abbrev.
+    size_t debug_abbrev_sz;       // Size of .debug_abbrev.
+
+    ptrdiff_t debug_info_off;     // Offset from this to .debug_info.
+    size_t debug_info_sz;         // Size of .debug_info.
+
+    ptrdiff_t activate_glue_off;
+    ptrdiff_t exit_task_glue_off;
+    ptrdiff_t unwind_glue_off;
+    ptrdiff_t yield_glue_off;
+
+public:
+
+    size_t n_rust_syms;
+    size_t n_c_syms;
+    size_t n_libs;
+
+    // Crates are immutable, constructed by the compiler.
+
+    uintptr_t get_image_base() const;
+    ptrdiff_t get_relocation_diff() const;
+    activate_glue_ty get_activate_glue() const;
+    uintptr_t get_exit_task_glue() const;
+    uintptr_t get_unwind_glue() const;
+    uintptr_t get_yield_glue() const;
+    struct mem_area
+    {
+      rust_dom *dom;
+      uintptr_t base;
+      uintptr_t lim;
+      mem_area(rust_dom *dom, uintptr_t pos, size_t sz);
+    };
+
+    mem_area get_debug_info(rust_dom *dom) const;
+    mem_area get_debug_abbrev(rust_dom *dom) const;
+};
+
+
+struct type_desc {
+    // First part of type_desc is known to compiler.
+    // first_param = &descs[1] if dynamic, null if static.
+    const type_desc **first_param;
+    size_t size;
+    size_t align;
+    uintptr_t copy_glue_off;
+    uintptr_t drop_glue_off;
+    uintptr_t free_glue_off;
+    uintptr_t mark_glue_off;     // For GC.
+    uintptr_t obj_drop_glue_off; // For custom destructors.
+
+    // Residual fields past here are known only to runtime.
+    UT_hash_handle hh;
+    size_t n_descs;
+    const type_desc *descs[];
+};
+
+class
+rust_crate_cache : public dom_owned<rust_crate_cache>,
+                   public rc_base<rust_crate_cache>
+{
+public:
+    class lib :
+        public rc_base<lib>, public dom_owned<lib>
+    {
+        uintptr_t handle;
+    public:
+        rust_dom *dom;
+        lib(rust_dom *dom, char const *name);
+        uintptr_t get_handle();
+        ~lib();
+    };
+
+    class c_sym :
+        public rc_base<c_sym>, public dom_owned<c_sym>
+    {
+        uintptr_t val;
+        lib *library;
+    public:
+        rust_dom *dom;
+        c_sym(rust_dom *dom, lib *library, char const *name);
+        uintptr_t get_val();
+        ~c_sym();
+    };
+
+    class rust_sym :
+        public rc_base<rust_sym>, public dom_owned<rust_sym>
+    {
+        uintptr_t val;
+        c_sym *crate_sym;
+    public:
+        rust_dom *dom;
+        rust_sym(rust_dom *dom, rust_crate const *curr_crate,
+                 c_sym *crate_sym, char const **path);
+        uintptr_t get_val();
+        ~rust_sym();
+    };
+
+    lib *get_lib(size_t n, char const *name);
+    c_sym *get_c_sym(size_t n, lib *library, char const *name);
+    rust_sym *get_rust_sym(size_t n,
+                           rust_dom *dom,
+                           rust_crate const *curr_crate,
+                           c_sym *crate_sym,
+                           char const **path);
+    type_desc *get_type_desc(size_t size,
+                             size_t align,
+                             size_t n_descs,
+                             type_desc const **descs);
+
+private:
+
+    rust_sym **rust_syms;
+    c_sym **c_syms;
+    lib **libs;
+    type_desc *type_descs;
+
+public:
+
+    rust_crate const *crate;
+    rust_dom *dom;
+    size_t idx;
+
+    rust_crate_cache(rust_dom *dom,
+                     rust_crate const *crate);
+    ~rust_crate_cache();
+    void flush();
+};
+
+#include "rust_dwarf.h"
+
+class
+rust_crate_reader
+{
+    struct mem_reader
+    {
+        rust_crate::mem_area &mem;
+        bool ok;
+        uintptr_t pos;
+
+        bool is_ok();
+        bool at_end();
+        void fail();
+        void reset();
+        mem_reader(rust_crate::mem_area &m);
+        size_t tell_abs();
+        size_t tell_off();
+        void seek_abs(uintptr_t p);
+        void seek_off(uintptr_t p);
+
+        template<typename T>
+        void get(T &out) {
+            if (pos < mem.base
+                || pos >= mem.lim
+                || pos + sizeof(T) > mem.lim)
+                ok = false;
+            if (!ok)
+                return;
+            out = *((T*)(pos));
+            pos += sizeof(T);
+            ok &= !at_end();
+            I(mem.dom, at_end() || (mem.base <= pos && pos < mem.lim));
+        }
+
+        template<typename T>
+        void get_uleb(T &out) {
+            out = T(0);
+            for (size_t i = 0; i < sizeof(T) && ok; ++i) {
+                uint8_t byte;
+                get(byte);
+                out <<= 7;
+                out |= byte & 0x7f;
+                if (!(byte & 0x80))
+                    break;
+            }
+            I(mem.dom, at_end() || (mem.base <= pos && pos < mem.lim));
+        }
+
+        template<typename T>
+        void adv_sizeof(T &) {
+            adv(sizeof(T));
+        }
+
+        bool adv_zstr(size_t sz);
+        bool get_zstr(char const *&c, size_t &sz);
+        void adv(size_t amt);
+    };
+
+    struct
+    abbrev : dom_owned<abbrev>
+    {
+        rust_dom *dom;
+        uintptr_t body_off;
+        size_t body_sz;
+        uintptr_t tag;
+        uint8_t has_children;
+        size_t idx;
+        abbrev(rust_dom *dom, uintptr_t body_off, size_t body_sz,
+               uintptr_t tag, uint8_t has_children);
+    };
+
+    class
+    abbrev_reader : public mem_reader
+    {
+        ptr_vec<abbrev> abbrevs;
+    public:
+        abbrev_reader(rust_crate::mem_area &abbrev_mem);
+        abbrev *get_abbrev(size_t i);
+        bool step_attr_form_pair(uintptr_t &attr, uintptr_t &form);
+        ~abbrev_reader();
+    };
+
+    rust_dom *dom;
+    size_t idx;
+    rust_crate const *crate;
+
+    rust_crate::mem_area abbrev_mem;
+    abbrev_reader abbrevs;
+
+    rust_crate::mem_area die_mem;
+
+public:
+
+    struct
+    attr
+    {
+        dw_form form;
+        dw_at at;
+        union {
+            struct {
+                char const *s;
+                size_t sz;
+            } str;
+            uintptr_t num;
+        } val;
+
+        bool is_numeric() const;
+        bool is_string() const;
+        size_t get_ssz(rust_dom *dom) const;
+        char const *get_str(rust_dom *dom) const;
+        uintptr_t get_num(rust_dom *dom) const;
+        bool is_unknown() const;
+    };
+
+    struct die_reader;
+
+    struct
+    die
+    {
+        die_reader *rdr;
+        uintptr_t off;
+        abbrev *ab;
+        bool using_rdr;
+
+        die(die_reader *rdr, uintptr_t off);
+        bool is_null() const;
+        bool has_children() const;
+        dw_tag tag() const;
+        bool start_attrs() const;
+        bool step_attr(attr &a) const;
+        bool find_str_attr(dw_at at, char const *&c);
+        bool find_num_attr(dw_at at, uintptr_t &n);
+        bool is_transparent();
+        bool find_child_by_name(char const *c, die &child,
+                                bool exact=false);
+        bool find_child_by_tag(dw_tag tag, die &child);
+        die next() const;
+        die next_sibling() const;
+    };
+
+    struct
+    rdr_sess
+    {
+        die_reader *rdr;
+        rdr_sess(die_reader *rdr);
+        ~rdr_sess();
+    };
+
+    struct
+    die_reader : public mem_reader
+    {
+        abbrev_reader &abbrevs;
+        uint32_t cu_unit_length;
+        uintptr_t cu_base;
+        uint16_t dwarf_vers;
+        uint32_t cu_abbrev_off;
+        uint8_t sizeof_addr;
+        bool in_use;
+
+        die first_die();
+        void dump();
+        die_reader(rust_crate::mem_area &die_mem,
+                   abbrev_reader &abbrevs);
+        ~die_reader();
+    };
+    die_reader dies;
+    rust_crate_reader(rust_dom *dom, rust_crate const *crate);
+};
+
+
+// A cond(ition) is something we can block on. This can be a channel
+// (writing), a port (reading) or a task (waiting).
+
+struct
+rust_cond
+{
+};
+
+// An alarm can be put into a wait queue and the task will be notified
+// when the wait queue is flushed.
+
+struct
+rust_alarm
+{
+    rust_task *receiver;
+    size_t idx;
+
+    rust_alarm(rust_task *receiver);
+};
+
+
+typedef ptr_vec<rust_alarm> rust_wait_queue;
+
+
+struct stk_seg {
+    unsigned int valgrind_id;
+    uintptr_t limit;
+    uint8_t data[];
+};
+
+struct frame_glue_fns {
+    uintptr_t mark_glue_off;
+    uintptr_t drop_glue_off;
+    uintptr_t reloc_glue_off;
+};
+
+struct
+rust_task : public rc_base<rust_task>,
+            public dom_owned<rust_task>,
+            public rust_cond
+{
+    // Fields known to the compiler.
+    stk_seg *stk;
+    uintptr_t runtime_sp;      // Runtime sp while task running.
+    uintptr_t rust_sp;         // Saved sp when not running.
+    uintptr_t gc_alloc_chain;  // Linked list of GC allocations.
+    rust_dom *dom;
+    rust_crate_cache *cache;
+
+    // Fields known only to the runtime.
+    ptr_vec<rust_task> *state;
+    rust_cond *cond;
+    uintptr_t* dptr;           // Rendezvous pointer for send/recv.
+    rust_task *spawner;        // Parent-link.
+    size_t idx;
+
+    // Wait queue for tasks waiting for this task.
+    rust_wait_queue waiting_tasks;
+    rust_alarm alarm;
+
+    rust_task(rust_dom *dom,
+              rust_task *spawner);
+    ~rust_task();
+
+    void start(uintptr_t exit_task_glue,
+               uintptr_t spawnee_fn,
+               uintptr_t args,
+               size_t callsz);
+    void grow(size_t n_frame_bytes);
+    bool running();
+    bool blocked();
+    bool blocked_on(rust_cond *cond);
+    bool dead();
+
+    const char *state_str();
+    void transition(ptr_vec<rust_task> *svec, ptr_vec<rust_task> *dvec);
+
+    void block(rust_cond *on);
+    void wakeup(rust_cond *from);
+    void die();
+    void unblock();
+
+    void check_active() { I(dom, dom->curr_task == this); }
+    void check_suspended() { I(dom, dom->curr_task != this); }
+
+    // Swap in some glue code to run when we have returned to the
+    // task's context (assuming we're the active task).
+    void run_after_return(size_t nargs, uintptr_t glue);
+
+    // Swap in some glue code to run when we're next activated
+    // (assuming we're the suspended task).
+    void run_on_resume(uintptr_t glue);
+
+    // Save callee-saved registers and return to the main loop.
+    void yield(size_t nargs);
+
+    // Fail this task (assuming caller-on-stack is different task).
+    void kill();
+
+    // Fail self, assuming caller-on-stack is this task.
+    void fail(size_t nargs);
+
+    // Notify tasks waiting for us that we are about to die.
+    void notify_waiting_tasks();
+
+    uintptr_t get_fp();
+    uintptr_t get_previous_fp(uintptr_t fp);
+    frame_glue_fns *get_frame_glue_fns(uintptr_t fp);
+    rust_crate_cache * get_crate_cache(rust_crate const *curr_crate);
+};
+
+struct rust_port : public rc_base<rust_port>,
+                   public task_owned<rust_port>,
+                   public rust_cond {
+    rust_task *task;
+    size_t unit_sz;
+    ptr_vec<rust_token> writers;
+    ptr_vec<rust_chan> chans;
+
+    rust_port(rust_task *task, size_t unit_sz);
+    ~rust_port();
+};
+
+struct rust_token : public rust_cond {
+    rust_chan *chan;      // Link back to the channel this token belongs to
+    size_t idx;           // Index into port->writers.
+    bool submitted;       // Whether token is in a port->writers.
+
+    rust_token(rust_chan *chan);
+    ~rust_token();
+
+    bool pending() const;
+    void submit();
+    void withdraw();
+};
+
+
+struct circ_buf : public dom_owned<circ_buf> {
+    static const size_t INIT_CIRC_BUF_UNITS = 8;
+    static const size_t MAX_CIRC_BUF_SIZE = 1 << 24;
+
+    rust_dom *dom;
+    size_t alloc;
+    size_t unit_sz;
+    size_t next;
+    size_t unread;
+    uint8_t *data;
+
+    circ_buf(rust_dom *dom, size_t unit_sz);
+    ~circ_buf();
+
+    void transfer(void *dst);
+    void push(void *src);
+    void shift(void *dst);
+};
+
+#include "rust_chan.h"
+
+int
+rust_main_loop(rust_dom *dom);
+
+//
+// Local Variables:
+// mode: C++
+// fill-column: 78;
+// indent-tabs-mode: nil
+// c-basic-offset: 4
+// buffer-file-coding-system: utf-8-unix
+// compile-command: "make -k -C .. 2>&1 | sed -e 's/\\/x\\//x:\\//g'";
+// End:
+//
+
+#endif
diff --git a/src/rt/rust_log.cpp b/src/rt/rust_log.cpp
new file mode 100644
index 00000000..102a2623
--- /dev/null
+++ b/src/rt/rust_log.cpp
@@ -0,0 +1,117 @@
+/*
+ * Logging infrastructure that aims to support multi-threading, indentation
+ * and ansi colors.
+ */
+
+#include "rust_internal.h"
+
+static uint32_t read_type_bit_mask() {
+    uint32_t bits = rust_log::ULOG | rust_log::ERR;
+    char *env_str = getenv("RUST_LOG");
+    if (env_str) {
+        bits = 0;
+        bits |= strstr(env_str, "err") ? rust_log::ERR : 0;
+        bits |= strstr(env_str, "mem") ? rust_log::MEM : 0;
+        bits |= strstr(env_str, "comm") ? rust_log::COMM : 0;
+        bits |= strstr(env_str, "task") ? rust_log::TASK : 0;
+        bits |= strstr(env_str, "up") ? rust_log::UPCALL : 0;
+        bits |= strstr(env_str, "dom") ? rust_log::DOM : 0;
+        bits |= strstr(env_str, "ulog") ? rust_log::ULOG : 0;
+        bits |= strstr(env_str, "trace") ? rust_log::TRACE : 0;
+        bits |= strstr(env_str, "dwarf") ? rust_log::DWARF : 0;
+        bits |= strstr(env_str, "cache") ? rust_log::CACHE : 0;
+        bits |= strstr(env_str, "timer") ? rust_log::TIMER : 0;
+        bits |= strstr(env_str, "all") ? rust_log::ALL : 0;
+    }
+    return bits;
+}
+
+rust_log::ansi_color rust_log::get_type_color(log_type type) {
+    switch (type) {
+    case ERR:
+        return rust_log::RED;
+    case UPCALL:
+        return rust_log::GREEN;
+    case COMM:
+        return rust_log::MAGENTA;
+    case DOM:
+    case TASK:
+        return rust_log::LIGHTTEAL;
+    case MEM:
+        return rust_log::YELLOW;
+    default:
+        return rust_log::WHITE;
+    }
+}
+
+static const char * _foreground_colors[] = { "[30m", "[1;30m", "[37m",
+                                             "[31m", "[1;31m", "[32m",
+                                             "[1;32m", "[33m", "[33m",
+                                             "[34m", "[1;34m", "[35m",
+                                             "[1;35m", "[36m", "[1;36m" };
+rust_log::rust_log(rust_srv *srv, rust_dom *dom) :
+    _srv(srv), _dom(dom), _type_bit_mask(read_type_bit_mask()),
+            _use_colors(getenv("RUST_COLOR_LOG")), _indent(0) {
+}
+
+rust_log::~rust_log() {
+
+}
+
+void rust_log::trace_ln(char *message) {
+    char buffer[512];
+    if (_use_colors) {
+        snprintf(buffer, sizeof(buffer), "\x1b%s0x%08" PRIxPTR "\x1b[0m: ",
+                 _foreground_colors[1 + ((uintptr_t) _dom % 2687 % (LIGHTTEAL
+                         - 1))], (uintptr_t) _dom);
+    } else {
+        snprintf(buffer, sizeof(buffer), "0x%08" PRIxPTR ": ",
+                 (uintptr_t) _dom);
+    }
+
+    for (uint32_t i = 0; i < _indent; i++) {
+        strncat(buffer, "\t", sizeof(buffer) - strlen(buffer) - 1);
+    }
+    strncat(buffer, message, sizeof(buffer) - strlen(buffer) - 1);
+    _srv->log(buffer);
+}
+
+/**
+ * Traces a log message if the specified logging type is not filtered.
+ */
+void rust_log::trace_ln(uint32_t type_bits, char *message) {
+    trace_ln(get_type_color((rust_log::log_type) type_bits), type_bits,
+             message);
+}
+
+/**
+ * Traces a log message using the specified ANSI color code.
+ */
+void rust_log::trace_ln(ansi_color color, uint32_t type_bits, char *message) {
+    if (is_tracing(type_bits)) {
+        if (_use_colors) {
+            char buffer[512];
+            snprintf(buffer, sizeof(buffer), "\x1b%s%s\x1b[0m",
+                     _foreground_colors[color], message);
+            trace_ln(buffer);
+        } else {
+            trace_ln(message);
+        }
+    }
+}
+
+bool rust_log::is_tracing(uint32_t type_bits) {
+    return type_bits & _type_bit_mask;
+}
+
+void rust_log::indent() {
+    _indent++;
+}
+
+void rust_log::outdent() {
+    _indent--;
+}
+
+void rust_log::reset_indent(uint32_t indent) {
+    _indent = indent;
+}
diff --git a/src/rt/rust_log.h b/src/rt/rust_log.h
new file mode 100644
index 00000000..b0c5fbec
--- /dev/null
+++ b/src/rt/rust_log.h
@@ -0,0 +1,59 @@
+#ifndef RUST_LOG_H_
+#define RUST_LOG_H_
+
+class rust_dom;
+
+class rust_log {
+    rust_srv *_srv;
+    rust_dom *_dom;
+    uint32_t _type_bit_mask;
+    bool _use_colors;
+    uint32_t _indent;
+    void trace_ln(char *message);
+public:
+    rust_log(rust_srv *srv, rust_dom *dom);
+    virtual ~rust_log();
+
+    enum ansi_color {
+        BLACK,
+        GRAY,
+        WHITE,
+        RED,
+        LIGHTRED,
+        GREEN,
+        LIGHTGREEN,
+        YELLOW,
+        LIGHTYELLOW,
+        BLUE,
+        LIGHTBLUE,
+        MAGENTA,
+        LIGHTMAGENTA,
+        TEAL,
+        LIGHTTEAL
+    };
+
+    enum log_type {
+        ERR = 0x1,
+        MEM = 0x2,
+        COMM = 0x4,
+        TASK = 0x8,
+        DOM = 0x10,
+        ULOG = 0x20,
+        TRACE = 0x40,
+        DWARF = 0x80,
+        CACHE = 0x100,
+        UPCALL = 0x200,
+        TIMER = 0x400,
+        ALL = 0xffffffff
+    };
+
+    void indent();
+    void outdent();
+    void reset_indent(uint32_t indent);
+    void trace_ln(uint32_t type_bits, char *message);
+    void trace_ln(ansi_color color, uint32_t type_bits, char *message);
+    bool is_tracing(uint32_t type_bits);
+    static ansi_color get_type_color(log_type type);
+};
+
+#endif /* RUST_LOG_H_ */
diff --git a/src/rt/rust_task.cpp b/src/rt/rust_task.cpp
new file mode 100644
index 00000000..beba11a0
--- /dev/null
+++ b/src/rt/rust_task.cpp
@@ -0,0 +1,474 @@
+
+#include "rust_internal.h"
+
+#include "valgrind.h"
+#include "memcheck.h"
+
+// Stacks
+
+static size_t const min_stk_bytes = 0x300;
+
+// Task stack segments. Heap allocated and chained together.
+
+static stk_seg*
+new_stk(rust_dom *dom, size_t minsz)
+{
+    if (minsz < min_stk_bytes)
+        minsz = min_stk_bytes;
+    size_t sz = sizeof(stk_seg) + minsz;
+    stk_seg *stk = (stk_seg *)dom->malloc(sz);
+    dom->logptr("new stk", (uintptr_t)stk);
+    memset(stk, 0, sizeof(stk_seg));
+    stk->limit = (uintptr_t) &stk->data[minsz];
+    dom->logptr("stk limit", stk->limit);
+    stk->valgrind_id =
+        VALGRIND_STACK_REGISTER(&stk->data[0],
+                                &stk->data[minsz]);
+    return stk;
+}
+
+static void
+del_stk(rust_dom *dom, stk_seg *stk)
+{
+    VALGRIND_STACK_DEREGISTER(stk->valgrind_id);
+    dom->logptr("freeing stk segment", (uintptr_t)stk);
+    dom->free(stk);
+}
+
+// Tasks
+
+// FIXME (issue #31): ifdef by platform. This is getting absurdly
+// x86-specific.
+
+size_t const n_callee_saves = 4;
+size_t const callee_save_fp = 0;
+
+static uintptr_t
+align_down(uintptr_t sp)
+{
+    // There is no platform we care about that needs more than a
+    // 16-byte alignment.
+    return sp & ~(16 - 1);
+}
+
+
+rust_task::rust_task(rust_dom *dom, rust_task *spawner) :
+    stk(new_stk(dom, 0)),
+    runtime_sp(0),
+    rust_sp(stk->limit),
+    gc_alloc_chain(0),
+    dom(dom),
+    cache(NULL),
+    state(&dom->running_tasks),
+    cond(NULL),
+    dptr(0),
+    spawner(spawner),
+    idx(0),
+    waiting_tasks(dom),
+    alarm(this)
+{
+    dom->logptr("new task", (uintptr_t)this);
+}
+
+rust_task::~rust_task()
+{
+    dom->log(rust_log::MEM|rust_log::TASK,
+             "~rust_task 0x%" PRIxPTR ", refcnt=%d",
+             (uintptr_t)this, refcnt);
+
+    /*
+      for (uintptr_t fp = get_fp(); fp; fp = get_previous_fp(fp)) {
+      frame_glue_fns *glue_fns = get_frame_glue_fns(fp);
+      dom->log(rust_log::MEM|rust_log::TASK,
+      "~rust_task, frame fp=0x%" PRIxPTR ", glue_fns=0x%" PRIxPTR,
+      fp, glue_fns);
+      if (glue_fns) {
+      dom->log(rust_log::MEM|rust_log::TASK,
+               "~rust_task, mark_glue=0x%" PRIxPTR,
+               glue_fns->mark_glue);
+      dom->log(rust_log::MEM|rust_log::TASK,
+               "~rust_task, drop_glue=0x%" PRIxPTR,
+               glue_fns->drop_glue);
+      dom->log(rust_log::MEM|rust_log::TASK,
+               "~rust_task, reloc_glue=0x%" PRIxPTR,
+               glue_fns->reloc_glue);
+      }
+      }
+    */
+
+    /* FIXME: tighten this up, there are some more
+       assertions that hold at task-lifecycle events. */
+    I(dom, refcnt == 0 ||
+      (refcnt == 1 && this == dom->root_task));
+
+    del_stk(dom, stk);
+    if (cache)
+        cache->deref();
+}
+
+void
+rust_task::start(uintptr_t exit_task_glue,
+                 uintptr_t spawnee_fn,
+                 uintptr_t args,
+                 size_t callsz)
+{
+    dom->logptr("exit-task glue", exit_task_glue);
+    dom->logptr("from spawnee", spawnee_fn);
+
+    // Set sp to last uintptr_t-sized cell of segment and align down.
+    rust_sp -= sizeof(uintptr_t);
+    rust_sp = align_down(rust_sp);
+
+    // Begin synthesizing frames. There are two: a "fully formed"
+    // exit-task frame at the top of the stack -- that pretends to be
+    // mid-execution -- and a just-starting frame beneath it that
+    // starts executing the first instruction of the spawnee. The
+    // spawnee *thinks* it was called by the exit-task frame above
+    // it. It wasn't; we put that fake frame in place here, but the
+    // illusion is enough for the spawnee to return to the exit-task
+    // frame when it's done, and exit.
+    uintptr_t *spp = (uintptr_t *)rust_sp;
+
+    // The exit_task_glue frame we synthesize above the frame we activate:
+    *spp-- = (uintptr_t) this;       // task
+    *spp-- = (uintptr_t) 0;          // output
+    *spp-- = (uintptr_t) 0;          // retpc
+    for (size_t j = 0; j < n_callee_saves; ++j) {
+        *spp-- = 0;
+    }
+
+    // We want 'frame_base' to point to the last callee-save in this
+    // (exit-task) frame, because we're going to inject this
+    // frame-pointer into the callee-save frame pointer value in the
+    // *next* (spawnee) frame. A cheap trick, but this means the
+    // spawnee frame will restore the proper frame pointer of the glue
+    // frame as it runs its epilogue.
+    uintptr_t frame_base = (uintptr_t) (spp+1);
+
+    *spp-- = (uintptr_t) dom->root_crate;  // crate ptr
+    *spp-- = (uintptr_t) 0;                // frame_glue_fns
+
+    // Copy args from spawner to spawnee.
+    if (args)  {
+        uintptr_t *src = (uintptr_t *)args;
+        src += 1;                  // spawn-call output slot
+        src += 1;                  // spawn-call task slot
+        // Memcpy all but the task and output pointers
+        callsz -= (2 * sizeof(uintptr_t));
+        spp = (uintptr_t*) (((uintptr_t)spp) - callsz);
+        memcpy(spp, src, callsz);
+
+        // Move sp down to point to task cell.
+        spp--;
+    } else {
+        // We're at root, starting up.
+        I(dom, callsz==0);
+    }
+
+    // The *implicit* incoming args to the spawnee frame we're
+    // activating:
+
+    *spp-- = (uintptr_t) this;            // task
+    *spp-- = (uintptr_t) 0;               // output addr
+    *spp-- = (uintptr_t) exit_task_glue;  // retpc
+
+    // The context the activate_glue needs to switch stack.
+    *spp-- = (uintptr_t) spawnee_fn;      // instruction to start at
+    for (size_t j = 0; j < n_callee_saves; ++j) {
+        // callee-saves to carry in when we activate
+        if (j == callee_save_fp)
+            *spp-- = frame_base;
+        else
+            *spp-- = NULL;
+    }
+
+    // Back up one, we overshot where sp should be.
+    rust_sp = (uintptr_t) (spp+1);
+
+    dom->add_task_to_state_vec(&dom->running_tasks, this);
+}
+
+void
+rust_task::grow(size_t n_frame_bytes)
+{
+    stk_seg *old_stk = this->stk;
+    uintptr_t old_top = (uintptr_t) old_stk->limit;
+    uintptr_t old_bottom = (uintptr_t) &old_stk->data[0];
+    uintptr_t rust_sp_disp = old_top - this->rust_sp;
+    size_t ssz = old_top - old_bottom;
+    dom->log(rust_log::MEM|rust_log::TASK|rust_log::UPCALL,
+             "upcall_grow_task(%" PRIdPTR
+             "), old size %" PRIdPTR
+             " bytes (old lim: 0x%" PRIxPTR ")",
+             n_frame_bytes, ssz, old_top);
+    ssz *= 2;
+    if (ssz < n_frame_bytes)
+        ssz = n_frame_bytes;
+    ssz = next_power_of_two(ssz);
+
+    dom->log(rust_log::MEM|rust_log::TASK, "upcall_grow_task growing stk 0x%"
+             PRIxPTR " to %d bytes", old_stk, ssz);
+
+    stk_seg *nstk = new_stk(dom, ssz);
+    uintptr_t new_top = (uintptr_t) &nstk->data[ssz];
+    size_t n_copy = old_top - old_bottom;
+    dom->log(rust_log::MEM|rust_log::TASK,
+             "copying %d bytes of stack from [0x%" PRIxPTR ", 0x%" PRIxPTR "]"
+             " to [0x%" PRIxPTR ", 0x%" PRIxPTR "]",
+             n_copy,
+             old_bottom, old_bottom + n_copy,
+             new_top - n_copy, new_top);
+
+    VALGRIND_MAKE_MEM_DEFINED((void*)old_bottom, n_copy);
+    memcpy((void*)(new_top - n_copy), (void*)old_bottom, n_copy);
+
+    nstk->limit = new_top;
+    this->stk = nstk;
+    this->rust_sp = new_top - rust_sp_disp;
+
+    dom->log(rust_log::MEM|rust_log::TASK, "processing relocations");
+
+    // FIXME (issue #32): this is the most ridiculously crude
+    // relocation scheme ever. Try actually, you know, writing out
+    // reloc descriptors?
+    size_t n_relocs = 0;
+    for (uintptr_t* p = (uintptr_t*)(new_top - n_copy);
+         p < (uintptr_t*)new_top; ++p) {
+        if (old_bottom <= *p && *p < old_top) {
+            //dom->log(rust_log::MEM, "relocating pointer 0x%" PRIxPTR
+            //        " by %d bytes", *p, (new_top - old_top));
+            n_relocs++;
+            *p += (new_top - old_top);
+        }
+    }
+    dom->log(rust_log::MEM|rust_log::TASK,
+             "processed %d relocations", n_relocs);
+    del_stk(dom, old_stk);
+    dom->logptr("grown stk limit", new_top);
+}
+
+void
+push_onto_thread_stack(uintptr_t &sp, uintptr_t value)
+{
+    asm("xchgl %0, %%esp\n"
+        "push %2\n"
+        "xchgl %0, %%esp\n"
+        : "=r" (sp)
+        : "0" (sp), "r" (value)
+        : "eax");
+}
+
+void
+rust_task::run_after_return(size_t nargs, uintptr_t glue)
+{
+    // This is only safe to call if we're the currently-running task.
+    check_active();
+
+    uintptr_t sp = runtime_sp;
+
+    // The compiler reserves nargs + 1 word for oldsp on the stack and
+    // then aligns it.
+    sp = align_down(sp - nargs * sizeof(uintptr_t));
+
+    uintptr_t *retpc = ((uintptr_t *) sp) - 1;
+    dom->log(rust_log::TASK|rust_log::MEM,
+             "run_after_return: overwriting retpc=0x%" PRIxPTR
+             " @ runtime_sp=0x%" PRIxPTR
+             " with glue=0x%" PRIxPTR,
+             *retpc, sp, glue);
+
+    // Move the current return address (which points into rust code)
+    // onto the rust stack and pretend we just called into the glue.
+    push_onto_thread_stack(rust_sp, *retpc);
+    *retpc = glue;
+}
+
+void
+rust_task::run_on_resume(uintptr_t glue)
+{
+    // This is only safe to call if we're suspended.
+    check_suspended();
+
+    // Inject glue as resume address in the suspended frame.
+    uintptr_t* rsp = (uintptr_t*) rust_sp;
+    rsp += n_callee_saves;
+    dom->log(rust_log::TASK|rust_log::MEM,
+             "run_on_resume: overwriting retpc=0x%" PRIxPTR
+             " @ rust_sp=0x%" PRIxPTR
+             " with glue=0x%" PRIxPTR,
+             *rsp, rsp, glue);
+    *rsp = glue;
+}
+
+void
+rust_task::yield(size_t nargs)
+{
+    dom->log(rust_log::TASK,
+             "task 0x%" PRIxPTR " yielding", this);
+    run_after_return(nargs, dom->root_crate->get_yield_glue());
+}
+
+static inline uintptr_t
+get_callee_save_fp(uintptr_t *top_of_callee_saves)
+{
+    return top_of_callee_saves[n_callee_saves - (callee_save_fp + 1)];
+}
+
+void
+rust_task::kill() {
+    // Note the distinction here: kill() is when you're in an upcall
+    // from task A and want to force-fail task B, you do B->kill().
+    // If you want to fail yourself you do self->fail(upcall_nargs).
+    dom->log(rust_log::TASK, "killing task 0x%" PRIxPTR, this);
+    // Unblock the task so it can unwind.
+    unblock();
+    if (this == dom->root_task)
+        dom->fail();
+    run_on_resume(dom->root_crate->get_unwind_glue());
+}
+
+void
+rust_task::fail(size_t nargs) {
+    // See note in ::kill() regarding who should call this.
+    dom->log(rust_log::TASK, "task 0x%" PRIxPTR " failing", this);
+    // Unblock the task so it can unwind.
+    unblock();
+    if (this == dom->root_task)
+        dom->fail();
+    run_after_return(nargs, dom->root_crate->get_unwind_glue());
+    if (spawner) {
+        dom->log(rust_log::TASK,
+                 "task 0x%" PRIxPTR
+                 " propagating failure to parent 0x%" PRIxPTR,
+                 this, spawner);
+        spawner->kill();
+    }
+}
+
+void
+rust_task::notify_waiting_tasks()
+{
+    while (waiting_tasks.length() > 0) {
+        rust_task *t = waiting_tasks.pop()->receiver;
+        if (!t->dead())
+            t->wakeup(this);
+    }
+}
+
+uintptr_t
+rust_task::get_fp() {
+    // sp in any suspended task points to the last callee-saved reg on
+    // the task stack.
+    return get_callee_save_fp((uintptr_t*)rust_sp);
+}
+
+uintptr_t
+rust_task::get_previous_fp(uintptr_t fp) {
+    // fp happens to, coincidentally (!) also point to the last
+    // callee-save on the task stack.
+    return get_callee_save_fp((uintptr_t*)fp);
+}
+
+frame_glue_fns*
+rust_task::get_frame_glue_fns(uintptr_t fp) {
+    fp -= sizeof(uintptr_t);
+    return *((frame_glue_fns**) fp);
+}
+
+bool
+rust_task::running()
+{
+    return state == &dom->running_tasks;
+}
+
+bool
+rust_task::blocked()
+{
+    return state == &dom->blocked_tasks;
+}
+
+bool
+rust_task::blocked_on(rust_cond *on)
+{
+    return blocked() && cond == on;
+}
+
+bool
+rust_task::dead()
+{
+    return state == &dom->dead_tasks;
+}
+
+void
+rust_task::transition(ptr_vec<rust_task> *src, ptr_vec<rust_task> *dst)
+{
+    I(dom, state == src);
+    dom->log(rust_log::TASK,
+             "task 0x%" PRIxPTR " state change '%s' -> '%s'",
+             (uintptr_t)this,
+             dom->state_vec_name(src),
+             dom->state_vec_name(dst));
+    dom->remove_task_from_state_vec(src, this);
+    dom->add_task_to_state_vec(dst, this);
+    state = dst;
+}
+
+void
+rust_task::block(rust_cond *on)
+{
+    I(dom, on);
+    transition(&dom->running_tasks, &dom->blocked_tasks);
+    dom->log(rust_log::TASK,
+             "task 0x%" PRIxPTR " blocking on 0x%" PRIxPTR,
+             (uintptr_t)this,
+             (uintptr_t)on);
+    cond = on;
+}
+
+void
+rust_task::wakeup(rust_cond *from)
+{
+    transition(&dom->blocked_tasks, &dom->running_tasks);
+    I(dom, cond == from);
+}
+
+void
+rust_task::die()
+{
+    transition(&dom->running_tasks, &dom->dead_tasks);
+}
+
+void
+rust_task::unblock()
+{
+    if (blocked())
+        wakeup(cond);
+}
+
+rust_crate_cache *
+rust_task::get_crate_cache(rust_crate const *curr_crate)
+{
+    if (cache && cache->crate != curr_crate) {
+        dom->log(rust_log::TASK, "switching task crate-cache to crate 0x%"
+                 PRIxPTR, curr_crate);
+        cache->deref();
+        cache = NULL;
+    }
+
+    if (!cache) {
+        dom->log(rust_log::TASK, "fetching cache for current crate");
+        cache = dom->get_cache(curr_crate);
+    }
+    return cache;
+}
+
+//
+// Local Variables:
+// mode: C++
+// fill-column: 78;
+// indent-tabs-mode: nil
+// c-basic-offset: 4
+// buffer-file-coding-system: utf-8-unix
+// compile-command: "make -k -C .. 2>&1 | sed -e 's/\\/x\\//x:\\//g'";
+// End:
+//
diff --git a/src/rt/rust_timer.cpp b/src/rt/rust_timer.cpp
new file mode 100644
index 00000000..897b7730
--- /dev/null
+++ b/src/rt/rust_timer.cpp
@@ -0,0 +1,97 @@
+
+#include "rust_internal.h"
+
+// The mechanism in this file is very crude; every domain (thread) spawns its
+// own secondary timer thread, and that timer thread *never idles*. It
+// sleep-loops interrupting the domain.
+//
+// This will need replacement, particularly in order to achieve an actual
+// state of idling when we're waiting on the outside world.  Though that might
+// be as simple as making a secondary waitable start/stop-timer signalling
+// system between the domain and its timer thread. We'll see.
+//
+// On the other hand, we don't presently have the ability to idle domains *at
+// all*, and without the timer thread we're unable to otherwise preempt rust
+// tasks. So ... one step at a time.
+//
+// The implementation here is "lockless" in the sense that it only involves
+// one-directional signaling of one-shot events, so the event initiator just
+// writes a nonzero word to a prederermined location and waits for the
+// receiver to see it show up in their memory.
+
+#if defined(__WIN32__)
+static DWORD WINAPI
+win32_timer_loop(void *ptr)
+{
+    // We were handed the rust_timer that owns us.
+    rust_timer *timer = (rust_timer *)ptr;
+    rust_dom &dom = timer->dom;
+    dom.log(LOG_TIMER, "in timer 0x%" PRIxPTR, (uintptr_t)timer);
+    while (!timer->exit_flag) {
+        Sleep(TIME_SLICE_IN_MS);
+        dom.log(LOG_TIMER,
+                "timer 0x%" PRIxPTR
+                " interrupting domain 0x%" PRIxPTR,
+                (uintptr_t)timer,
+                (uintptr_t)&dom);
+        dom.interrupt_flag = 1;
+    }
+    ExitThread(0);
+    return 0;
+}
+
+#elif defined(__GNUC__)
+static void *
+pthread_timer_loop(void *ptr)
+{
+    // We were handed the rust_timer that owns us.
+    rust_timer *timer = (rust_timer *)ptr;
+    rust_dom &dom(timer->dom);
+    while (!timer->exit_flag) {
+        usleep(TIME_SLICE_IN_MS * 1000);
+        dom.interrupt_flag = 1;
+    }
+    pthread_exit(NULL);
+    return 0;
+
+}
+#else
+#error "Platform not supported"
+#endif
+
+
+rust_timer::rust_timer(rust_dom &dom) : dom(dom), exit_flag(0)
+{
+    dom.log(rust_log::TIMER, "creating timer for domain 0x%" PRIxPTR, &dom);
+#if defined(__WIN32__)
+    thread = CreateThread(NULL, 0, win32_timer_loop, this, 0, NULL);
+    dom.win32_require("CreateThread", thread != NULL);
+#else
+    pthread_attr_init(&attr);
+    pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
+    pthread_create(&thread, &attr, pthread_timer_loop, (void *)this);
+#endif
+}
+
+rust_timer::~rust_timer()
+{
+    exit_flag = 1;
+#if defined(__WIN32__)
+    dom.win32_require("WaitForSingleObject",
+                      WaitForSingleObject(thread, INFINITE)
+                      == WAIT_OBJECT_0);
+#else
+    pthread_join(thread, NULL);
+#endif
+}
+
+//
+// Local Variables:
+// mode: C++
+// fill-column: 78;
+// indent-tabs-mode: nil
+// c-basic-offset: 4
+// buffer-file-coding-system: utf-8-unix
+// compile-command: "make -k -C .. 2>&1 | sed -e 's/\\/x\\//x:\\//g'";
+// End:
+//
diff --git a/src/rt/rust_upcall.cpp b/src/rt/rust_upcall.cpp
new file mode 100644
index 00000000..3a17ea1c
--- /dev/null
+++ b/src/rt/rust_upcall.cpp
@@ -0,0 +1,654 @@
+
+#include "rust_internal.h"
+
+
+// Upcalls.
+
+#ifdef __GNUC__
+#define LOG_UPCALL_ENTRY(task)                              \
+    (task)->dom->get_log().reset_indent(0);                 \
+    (task)->dom->log(rust_log::UPCALL,                      \
+                     "upcall task: 0x%" PRIxPTR             \
+                     " retpc: 0x%" PRIxPTR,                 \
+                     (task), __builtin_return_address(0));  \
+    (task)->dom->get_log().indent();
+#else
+#define LOG_UPCALL_ENTRY(task)                              \
+    (task)->dom->get_log().reset_indent(0);                 \
+    (task)->dom->log(rust_log::UPCALL,                      \
+                     "upcall task: 0x%" PRIxPTR (task));    \
+    (task)->dom->get_log().indent();
+#endif
+
+extern "C" CDECL char const *str_buf(rust_task *task, rust_str *s);
+
+extern "C" void
+upcall_grow_task(rust_task *task, size_t n_frame_bytes)
+{
+    LOG_UPCALL_ENTRY(task);
+    task->grow(n_frame_bytes);
+}
+
+extern "C" CDECL void
+upcall_log_int(rust_task *task, int32_t i)
+{
+    LOG_UPCALL_ENTRY(task);
+    task->dom->log(rust_log::UPCALL|rust_log::ULOG,
+                   "upcall log_int(0x%" PRIx32 " = %" PRId32 " = '%c')",
+                   i, i, (char)i);
+}
+
+extern "C" CDECL void
+upcall_log_str(rust_task *task, rust_str *str)
+{
+    LOG_UPCALL_ENTRY(task);
+    const char *c = str_buf(task, str);
+    task->dom->log(rust_log::UPCALL|rust_log::ULOG,
+                   "upcall log_str(\"%s\")",
+                   c);
+}
+
+extern "C" CDECL void
+upcall_trace_word(rust_task *task, uintptr_t i)
+{
+    LOG_UPCALL_ENTRY(task);
+    task->dom->log(rust_log::UPCALL|rust_log::TRACE,
+                   "trace: 0x%" PRIxPTR "",
+                   i, i, (char)i);
+}
+
+extern "C" CDECL void
+upcall_trace_str(rust_task *task, char const *c)
+{
+    LOG_UPCALL_ENTRY(task);
+    task->dom->log(rust_log::UPCALL|rust_log::TRACE,
+                   "trace: %s",
+                   c);
+}
+
+extern "C" CDECL rust_port*
+upcall_new_port(rust_task *task, size_t unit_sz)
+{
+    LOG_UPCALL_ENTRY(task);
+    rust_dom *dom = task->dom;
+    dom->log(rust_log::UPCALL|rust_log::MEM|rust_log::COMM,
+             "upcall_new_port(task=0x%" PRIxPTR ", unit_sz=%d)",
+             (uintptr_t)task, unit_sz);
+    return new (dom) rust_port(task, unit_sz);
+}
+
+extern "C" CDECL void
+upcall_del_port(rust_task *task, rust_port *port)
+{
+    LOG_UPCALL_ENTRY(task);
+    task->dom->log(rust_log::UPCALL|rust_log::MEM|rust_log::COMM,
+                   "upcall del_port(0x%" PRIxPTR ")", (uintptr_t)port);
+    I(task->dom, !port->refcnt);
+    delete port;
+}
+
+extern "C" CDECL rust_chan*
+upcall_new_chan(rust_task *task, rust_port *port)
+{
+    LOG_UPCALL_ENTRY(task);
+    rust_dom *dom = task->dom;
+    dom->log(rust_log::UPCALL|rust_log::MEM|rust_log::COMM,
+             "upcall_new_chan(task=0x%" PRIxPTR ", port=0x%" PRIxPTR ")",
+             (uintptr_t)task, port);
+    I(dom, port);
+    return new (dom) rust_chan(task, port);
+}
+
+extern "C" CDECL void
+upcall_del_chan(rust_task *task, rust_chan *chan)
+{
+    LOG_UPCALL_ENTRY(task);
+    rust_dom *dom = task->dom;
+    dom->log(rust_log::UPCALL|rust_log::MEM|rust_log::COMM,
+             "upcall del_chan(0x%" PRIxPTR ")", (uintptr_t)chan);
+    I(dom, !chan->refcnt);
+    delete chan;
+}
+
+extern "C" CDECL rust_chan *
+upcall_clone_chan(rust_task *task, rust_task *owner, rust_chan *chan)
+{
+    LOG_UPCALL_ENTRY(task);
+    rust_dom *dom = task->dom;
+    dom->log(rust_log::UPCALL|rust_log::MEM|rust_log::COMM,
+             "upcall clone_chan(owner 0x%" PRIxPTR ", chan 0x%" PRIxPTR ")",
+             (uintptr_t)owner, (uintptr_t)chan);
+    return new (owner->dom) rust_chan(owner, chan->port);
+}
+
+
+/*
+ * Buffering protocol:
+ *
+ *   - Reader attempts to read:
+ *     - Set reader to blocked-reading state.
+ *     - If buf with data exists:
+ *       - Attempt transmission.
+ *
+ *  - Writer attempts to write:
+ *     - Set writer to blocked-writing state.
+ *     - Copy data into chan.
+ *     - Attempt transmission.
+ *
+ *  - Transmission:
+ *       - Copy data from buf to reader
+ *       - Decr buf
+ *       - Set reader to running
+ *       - If buf now empty and blocked writer:
+ *         - Set blocked writer to running
+ *
+ */
+
+static int
+attempt_transmission(rust_dom *dom,
+                     rust_chan *src,
+                     rust_task *dst)
+{
+    I(dom, src);
+    I(dom, dst);
+
+    rust_port *port = src->port;
+    if (!port) {
+        dom->log(rust_log::COMM,
+                 "src died, transmission incomplete");
+        return 0;
+    }
+
+    circ_buf *buf = &src->buffer;
+    if (buf->unread == 0) {
+        dom->log(rust_log::COMM,
+                 "buffer empty, transmission incomplete");
+        return 0;
+    }
+
+    if (!dst->blocked_on(port)) {
+        dom->log(rust_log::COMM,
+                 "dst in non-reading state, transmission incomplete");
+        return 0;
+    }
+
+    uintptr_t *dptr = dst->dptr;
+    dom->log(rust_log::COMM,
+             "receiving %d bytes into dst_task=0x%" PRIxPTR
+             ", dptr=0x%" PRIxPTR,
+             port->unit_sz, dst, dptr);
+    buf->shift(dptr);
+
+    // Wake up the sender if its waiting for the send operation.
+    rust_task *sender = src->task;
+    rust_token *token = &src->token;
+    if (sender->blocked_on(token))
+        sender->wakeup(token);
+
+    // Wake up the receiver, there is new data.
+    dst->wakeup(port);
+
+    dom->log(rust_log::COMM, "transmission complete");
+    return 1;
+}
+
+extern "C" CDECL void
+upcall_yield(rust_task *task)
+{
+    LOG_UPCALL_ENTRY(task);
+    rust_dom *dom = task->dom;
+    dom->log(rust_log::UPCALL|rust_log::COMM, "upcall yield()");
+    task->yield(1);
+}
+
+extern "C" CDECL void
+upcall_join(rust_task *task, rust_task *other)
+{
+    LOG_UPCALL_ENTRY(task);
+    rust_dom *dom = task->dom;
+    dom->log(rust_log::UPCALL|rust_log::COMM,
+             "upcall join(other=0x%" PRIxPTR ")",
+             (uintptr_t)other);
+
+    // If the other task is already dying, we dont have to wait for it.
+    if (!other->dead()) {
+        other->waiting_tasks.push(&task->alarm);
+        task->block(other);
+        task->yield(2);
+    }
+}
+
+extern "C" CDECL void
+upcall_send(rust_task *task, rust_chan *chan, void *sptr)
+{
+    LOG_UPCALL_ENTRY(task);
+    rust_dom *dom = task->dom;
+    dom->log(rust_log::UPCALL|rust_log::COMM,
+             "upcall send(chan=0x%" PRIxPTR ", sptr=0x%" PRIxPTR ")",
+             (uintptr_t)chan,
+             (uintptr_t)sptr);
+
+    I(dom, chan);
+    I(dom, sptr);
+
+    rust_port *port = chan->port;
+    dom->log(rust_log::MEM|rust_log::COMM,
+             "send to port", (uintptr_t)port);
+    I(dom, port);
+
+    rust_token *token = &chan->token;
+    dom->log(rust_log::MEM|rust_log::COMM,
+             "sending via token 0x%" PRIxPTR,
+             (uintptr_t)token);
+
+    if (port->task) {
+        chan->buffer.push(sptr);
+        task->block(token);
+        attempt_transmission(dom, chan, port->task);
+        if (chan->buffer.unread && !token->pending())
+            token->submit();
+    } else {
+        dom->log(rust_log::COMM|rust_log::ERR,
+                 "port has no task (possibly throw?)");
+    }
+
+    if (!task->running())
+        task->yield(3);
+}
+
+extern "C" CDECL void
+upcall_recv(rust_task *task, uintptr_t *dptr, rust_port *port)
+{
+    LOG_UPCALL_ENTRY(task);
+    rust_dom *dom = task->dom;
+    dom->log(rust_log::UPCALL|rust_log::COMM,
+             "upcall recv(dptr=0x" PRIxPTR ", port=0x%" PRIxPTR ")",
+             (uintptr_t)dptr,
+             (uintptr_t)port);
+
+    I(dom, port);
+    I(dom, port->task);
+    I(dom, task);
+    I(dom, port->task == task);
+
+    task->block(port);
+
+    if (port->writers.length() > 0) {
+        I(dom, task->dom);
+        size_t i = rand(&dom->rctx);
+        i %= port->writers.length();
+        rust_token *token = port->writers[i];
+        rust_chan *chan = token->chan;
+        if (attempt_transmission(dom, chan, task))
+            token->withdraw();
+    } else {
+        dom->log(rust_log::COMM,
+                 "no writers sending to port", (uintptr_t)port);
+    }
+
+    if (!task->running()) {
+        task->dptr = dptr;
+        task->yield(3);
+    }
+}
+
+extern "C" CDECL void
+upcall_fail(rust_task *task, char const *expr, char const *file, size_t line)
+{
+    LOG_UPCALL_ENTRY(task);
+    task->dom->log(rust_log::UPCALL|rust_log::ERR,
+                   "upcall fail '%s', %s:%" PRIdPTR,
+                   expr, file, line);
+    task->fail(4);
+}
+
+extern "C" CDECL void
+upcall_kill(rust_task *task, rust_task *target)
+{
+    LOG_UPCALL_ENTRY(task);
+    task->dom->log(rust_log::UPCALL|rust_log::TASK,
+                   "upcall kill target=0x%" PRIxPTR, target);
+    target->kill();
+}
+
+extern "C" CDECL void
+upcall_exit(rust_task *task)
+{
+    LOG_UPCALL_ENTRY(task);
+
+    rust_dom *dom = task->dom;
+    dom->log(rust_log::UPCALL|rust_log::TASK, "upcall exit");
+    task->die();
+    task->notify_waiting_tasks();
+    task->yield(1);
+}
+
+extern "C" CDECL uintptr_t
+upcall_malloc(rust_task *task, size_t nbytes)
+{
+    LOG_UPCALL_ENTRY(task);
+
+    void *p = task->dom->malloc(nbytes);
+    task->dom->log(rust_log::UPCALL|rust_log::MEM,
+                   "upcall malloc(%u) = 0x%" PRIxPTR,
+                   nbytes, (uintptr_t)p);
+    return (uintptr_t) p;
+}
+
+extern "C" CDECL void
+upcall_free(rust_task *task, void* ptr)
+{
+    LOG_UPCALL_ENTRY(task);
+
+    rust_dom *dom = task->dom;
+    dom->log(rust_log::UPCALL|rust_log::MEM,
+             "upcall free(0x%" PRIxPTR ")",
+             (uintptr_t)ptr);
+    dom->free(ptr);
+}
+
+extern "C" CDECL rust_str *
+upcall_new_str(rust_task *task, char const *s, size_t fill)
+{
+    LOG_UPCALL_ENTRY(task);
+    rust_dom *dom = task->dom;
+    dom->log(rust_log::UPCALL|rust_log::MEM,
+             "upcall new_str('%s', %" PRIdPTR ")", s, fill);
+    size_t alloc = next_power_of_two(sizeof(rust_str) + fill);
+    void *mem = dom->malloc(alloc);
+    if (!mem) {
+        task->fail(3);
+        return NULL;
+    }
+    rust_str *st = new (mem) rust_str(dom, alloc, fill, (uint8_t const *)s);
+    dom->log(rust_log::UPCALL|rust_log::MEM,
+             "upcall new_str('%s', %" PRIdPTR ") = 0x%" PRIxPTR,
+             s, fill, st);
+    return st;
+}
+
+extern "C" CDECL rust_vec *
+upcall_new_vec(rust_task *task, size_t fill)
+{
+    LOG_UPCALL_ENTRY(task);
+    rust_dom *dom = task->dom;
+    dom->log(rust_log::UPCALL|rust_log::MEM,
+             "upcall new_vec(%" PRIdPTR ")", fill);
+    size_t alloc = next_power_of_two(sizeof(rust_vec) + fill);
+    void *mem = dom->malloc(alloc);
+    if (!mem) {
+        task->fail(3);
+        return NULL;
+    }
+    rust_vec *v = new (mem) rust_vec(dom, alloc, 0, NULL);
+    dom->log(rust_log::UPCALL|rust_log::MEM,
+             "upcall new_vec(%" PRIdPTR ") = 0x%" PRIxPTR,
+             fill, v);
+    return v;
+}
+
+
+extern "C" CDECL rust_str *
+upcall_vec_grow(rust_task *task, rust_vec *v, size_t n_bytes)
+{
+    LOG_UPCALL_ENTRY(task);
+    rust_dom *dom = task->dom;
+    dom->log(rust_log::UPCALL|rust_log::MEM,
+             "upcall vec_grow(%" PRIxPTR ", %" PRIdPTR ")", v, n_bytes);
+    size_t alloc = next_power_of_two(sizeof(rust_vec) + v->fill + n_bytes);
+    if (v->refcnt == 1) {
+
+        // Fastest path: already large enough.
+        if (v->alloc >= alloc) {
+            dom->log(rust_log::UPCALL|rust_log::MEM, "no-growth path");
+            return v;
+        }
+
+        // Second-fastest path: can at least realloc.
+        dom->log(rust_log::UPCALL|rust_log::MEM, "realloc path");
+        v = (rust_vec*)dom->realloc(v, alloc);
+        if (!v) {
+            task->fail(3);
+            return NULL;
+        }
+        v->alloc = alloc;
+
+    } else {
+        // Slowest path: make a new vec.
+        dom->log(rust_log::UPCALL|rust_log::MEM, "new vec path");
+        void *mem = dom->malloc(alloc);
+        if (!mem) {
+            task->fail(3);
+            return NULL;
+        }
+        v->deref();
+        v = new (mem) rust_vec(dom, alloc, v->fill, &v->data[0]);
+    }
+    I(dom, sizeof(rust_vec) + v->fill <= v->alloc);
+    return v;
+}
+
+
+static rust_crate_cache::c_sym *
+fetch_c_sym(rust_task *task,
+            rust_crate const *curr_crate,
+            size_t lib_num,
+            size_t c_sym_num,
+            char const *library,
+            char const *symbol)
+{
+    rust_crate_cache *cache = task->get_crate_cache(curr_crate);
+    rust_crate_cache::lib *l = cache->get_lib(lib_num, library);
+    return cache->get_c_sym(c_sym_num, l, symbol);
+}
+
+extern "C" CDECL uintptr_t
+upcall_require_rust_sym(rust_task *task,
+                        rust_crate const *curr_crate,
+                        size_t lib_num,      // # of lib
+                        size_t c_sym_num,    // # of C sym "rust_crate" in lib
+                        size_t rust_sym_num, // # of rust sym
+                        char const *library,
+                        char const **path)
+{
+    LOG_UPCALL_ENTRY(task);
+    rust_dom *dom = task->dom;
+
+    dom->log(rust_log::UPCALL|rust_log::CACHE,
+             "upcall require rust sym: lib #%" PRIdPTR
+             " = %s, c_sym #%" PRIdPTR
+             ", rust_sym #%" PRIdPTR
+             ", curr_crate = 0x%" PRIxPTR,
+             lib_num, library, c_sym_num, rust_sym_num,
+             curr_crate);
+    for (char const **c = crate_rel(curr_crate, path); *c; ++c) {
+        dom->log(rust_log::UPCALL, " + %s", crate_rel(curr_crate, *c));
+    }
+
+    dom->log(rust_log::UPCALL|rust_log::CACHE,
+             "require C symbol 'rust_crate' from lib #%" PRIdPTR,lib_num);
+    rust_crate_cache::c_sym *c =
+        fetch_c_sym(task, curr_crate, lib_num, c_sym_num,
+                    library, "rust_crate");
+
+    dom->log(rust_log::UPCALL|rust_log::CACHE,
+             "require rust symbol inside crate");
+    rust_crate_cache::rust_sym *s =
+        task->cache->get_rust_sym(rust_sym_num, dom, curr_crate, c, path);
+
+    uintptr_t addr = s->get_val();
+    if (addr) {
+        dom->log(rust_log::UPCALL|rust_log::CACHE,
+                 "found-or-cached addr: 0x%" PRIxPTR, addr);
+    } else {
+        dom->log(rust_log::UPCALL|rust_log::CACHE,
+                 "failed to resolve symbol");
+        task->fail(7);
+    }
+    return addr;
+}
+
+extern "C" CDECL uintptr_t
+upcall_require_c_sym(rust_task *task,
+                     rust_crate const *curr_crate,
+                     size_t lib_num,      // # of lib
+                     size_t c_sym_num,    // # of C sym
+                     char const *library,
+                     char const *symbol)
+{
+    LOG_UPCALL_ENTRY(task);
+    rust_dom *dom = task->dom;
+
+    dom->log(rust_log::UPCALL|rust_log::CACHE,
+             "upcall require c sym: lib #%" PRIdPTR
+             " = %s, c_sym #%" PRIdPTR
+             " = %s"
+             ", curr_crate = 0x%" PRIxPTR,
+             lib_num, library, c_sym_num, symbol, curr_crate);
+
+    rust_crate_cache::c_sym *c =
+        fetch_c_sym(task, curr_crate, lib_num, c_sym_num, library, symbol);
+
+    uintptr_t addr = c->get_val();
+    if (addr) {
+        dom->log(rust_log::UPCALL|rust_log::CACHE,
+                 "found-or-cached addr: 0x%" PRIxPTR, addr);
+    } else {
+        dom->log(rust_log::UPCALL|rust_log::CACHE,
+                 "failed to resolve symbol");
+        task->fail(6);
+    }
+    return addr;
+}
+
+extern "C" CDECL type_desc *
+upcall_get_type_desc(rust_task *task,
+                     rust_crate const *curr_crate,
+                     size_t size,
+                     size_t align,
+                     size_t n_descs,
+                     type_desc const **descs)
+{
+    LOG_UPCALL_ENTRY(task);
+    rust_dom *dom = task->dom;
+    dom->log(rust_log::UPCALL|rust_log::CACHE,
+             "upcall get_type_desc with size=%" PRIdPTR
+             ", align=%" PRIdPTR ", %" PRIdPTR " descs",
+             size, align, n_descs);
+    rust_crate_cache *cache = task->get_crate_cache(curr_crate);
+    type_desc *td = cache->get_type_desc(size, align, n_descs, descs);
+    dom->log(rust_log::UPCALL|rust_log::CACHE,
+             "returning tydesc 0x%" PRIxPTR, td);
+    return td;
+}
+
+
+#if defined(__WIN32__)
+static DWORD WINAPI rust_thread_start(void *ptr)
+#elif defined(__GNUC__)
+static void *rust_thread_start(void *ptr)
+#else
+#error "Platform not supported"
+#endif
+{
+    // We were handed the domain we are supposed to run.
+    rust_dom *dom = (rust_dom *)ptr;
+
+    // Start a new rust main loop for this thread.
+    rust_main_loop(dom);
+
+    rust_srv *srv = dom->srv;
+    delete dom;
+    delete srv;
+
+    return 0;
+}
+
+extern "C" CDECL rust_task *
+upcall_new_task(rust_task *spawner)
+{
+    LOG_UPCALL_ENTRY(spawner);
+
+    rust_dom *dom = spawner->dom;
+    rust_task *task = new (dom) rust_task(dom, spawner);
+    dom->log(rust_log::UPCALL|rust_log::MEM|rust_log::TASK,
+             "upcall new_task(spawner 0x%" PRIxPTR ") = 0x%" PRIxPTR,
+             spawner, task);
+    return task;
+}
+
+extern "C" CDECL rust_task *
+upcall_start_task(rust_task *spawner,
+                  rust_task *task,
+                  uintptr_t exit_task_glue,
+                  uintptr_t spawnee_fn,
+                  size_t callsz)
+{
+    LOG_UPCALL_ENTRY(spawner);
+
+    rust_dom *dom = spawner->dom;
+    dom->log(rust_log::UPCALL|rust_log::MEM|rust_log::TASK,
+             "upcall start_task(task 0x%" PRIxPTR
+             " exit_task_glue 0x%" PRIxPTR
+             ", spawnee 0x%" PRIxPTR
+             ", callsz %" PRIdPTR ")",
+             task, exit_task_glue, spawnee_fn, callsz);
+    task->start(exit_task_glue, spawnee_fn, spawner->rust_sp, callsz);
+    return task;
+}
+
+extern "C" CDECL rust_task *
+upcall_new_thread(rust_task *task)
+{
+    LOG_UPCALL_ENTRY(task);
+
+    rust_dom *old_dom = task->dom;
+    rust_dom *new_dom = new rust_dom(old_dom->srv->clone(),
+                                     old_dom->root_crate);
+    new_dom->log(rust_log::UPCALL|rust_log::MEM,
+                 "upcall new_thread() = 0x%" PRIxPTR,
+                 new_dom->root_task);
+    return new_dom->root_task;
+}
+
+extern "C" CDECL rust_task *
+upcall_start_thread(rust_task *spawner,
+                    rust_task *root_task,
+                    uintptr_t exit_task_glue,
+                    uintptr_t spawnee_fn,
+                    size_t callsz)
+{
+    LOG_UPCALL_ENTRY(spawner);
+
+    rust_dom *dom = spawner->dom;
+    dom->log(rust_log::UPCALL|rust_log::MEM|rust_log::TASK,
+             "upcall start_thread(exit_task_glue 0x%" PRIxPTR
+             ", spawnee 0x%" PRIxPTR
+             ", callsz %" PRIdPTR ")",
+             exit_task_glue, spawnee_fn, callsz);
+    root_task->start(exit_task_glue, spawnee_fn, spawner->rust_sp, callsz);
+
+#if defined(__WIN32__)
+    HANDLE thread;
+    thread = CreateThread(NULL, 0, rust_thread_start, root_task->dom,
+                          0, NULL);
+    dom->win32_require("CreateThread", thread != NULL);
+#else
+    pthread_t thread;
+    pthread_create(&thread, &dom->attr, rust_thread_start,
+                   (void *)root_task->dom);
+#endif
+
+    return 0;
+}
+
+//
+// Local Variables:
+// mode: C++
+// fill-column: 78;
+// indent-tabs-mode: nil
+// c-basic-offset: 4
+// buffer-file-coding-system: utf-8-unix
+// compile-command: "make -k -C .. 2>&1 | sed -e 's/\\/x\\//x:\\//g'";
+// End:
+//
diff --git a/src/rt/rust_util.h b/src/rt/rust_util.h
new file mode 100644
index 00000000..6f34dad9
--- /dev/null
+++ b/src/rt/rust_util.h
@@ -0,0 +1,155 @@
+#ifndef RUST_UTIL_H
+#define RUST_UTIL_H
+
+// Reference counted objects
+
+template <typename T>
+rc_base<T>::rc_base() :
+    refcnt(1)
+{
+}
+
+template <typename T>
+rc_base<T>::~rc_base()
+{
+}
+
+// Utility type: pointer-vector.
+
+template <typename T>
+ptr_vec<T>::ptr_vec(rust_dom *dom) :
+    dom(dom),
+    alloc(INIT_SIZE),
+    fill(0),
+    data(new (dom) T*[alloc])
+{
+    I(dom, data);
+    dom->log(rust_log::MEM,
+             "new ptr_vec(data=0x%" PRIxPTR ") -> 0x%" PRIxPTR,
+             (uintptr_t)data, (uintptr_t)this);
+}
+
+template <typename T>
+ptr_vec<T>::~ptr_vec()
+{
+    I(dom, data);
+    dom->log(rust_log::MEM,
+             "~ptr_vec 0x%" PRIxPTR ", data=0x%" PRIxPTR,
+             (uintptr_t)this, (uintptr_t)data);
+    I(dom, fill == 0);
+    dom->free(data);
+}
+
+template <typename T> T *&
+ptr_vec<T>::operator[](size_t offset) {
+    I(dom, data[offset]->idx == offset);
+    return data[offset];
+}
+
+template <typename T>
+void
+ptr_vec<T>::push(T *p)
+{
+    I(dom, data);
+    I(dom, fill <= alloc);
+    if (fill == alloc) {
+        alloc *= 2;
+        data = (T **)dom->realloc(data, alloc * sizeof(T*));
+        I(dom, data);
+    }
+    I(dom, fill < alloc);
+    p->idx = fill;
+    data[fill++] = p;
+}
+
+template <typename T>
+T *
+ptr_vec<T>::pop()
+{
+    return data[--fill];
+}
+
+template <typename T>
+void
+ptr_vec<T>::trim(size_t sz)
+{
+    I(dom, data);
+    if (sz <= (alloc / 4) &&
+        (alloc / 2) >= INIT_SIZE) {
+        alloc /= 2;
+        I(dom, alloc >= fill);
+        data = (T **)dom->realloc(data, alloc * sizeof(T*));
+        I(dom, data);
+    }
+}
+
+template <typename T>
+void
+ptr_vec<T>::swapdel(T *item)
+{
+    /* Swap the endpoint into i and decr fill. */
+    I(dom, data);
+    I(dom, fill > 0);
+    I(dom, item->idx < fill);
+    fill--;
+    if (fill > 0) {
+        T *subst = data[fill];
+        size_t idx = item->idx;
+        data[idx] = subst;
+        subst->idx = idx;
+    }
+}
+
+// Inline fn used regularly elsewhere.
+
+static inline size_t
+next_power_of_two(size_t s)
+{
+    size_t tmp = s - 1;
+    tmp |= tmp >> 1;
+    tmp |= tmp >> 2;
+    tmp |= tmp >> 4;
+    tmp |= tmp >> 8;
+    tmp |= tmp >> 16;
+#if SIZE_MAX == UINT64_MAX
+    tmp |= tmp >> 32;
+#endif
+    return tmp + 1;
+}
+
+// Vectors (rust-user-code level).
+
+struct
+rust_vec : public rc_base<rust_vec>
+{
+    size_t alloc;
+    size_t fill;
+    uint8_t data[];
+    rust_vec(rust_dom *dom, size_t alloc, size_t fill, uint8_t const *d) :
+        alloc(alloc),
+        fill(fill)
+    {
+        if (d || fill) {
+            I(dom, d);
+            I(dom, fill);
+            memcpy(&data[0], d, fill);
+        }
+    }
+    ~rust_vec() {}
+};
+
+// Rust types vec and str look identical from our perspective.
+typedef rust_vec rust_str;
+
+//
+// Local Variables:
+// mode: C++
+// fill-column: 78;
+// indent-tabs-mode: nil
+// c-basic-offset: 4
+// buffer-file-coding-system: utf-8-unix
+// compile-command: "make -k -C .. 2>&1 | sed -e 's/\\/x\\//x:\\//g'";
+// End:
+//
+
+#endif
diff --git a/src/rt/sync/fair_ticket_lock.cpp b/src/rt/sync/fair_ticket_lock.cpp
new file mode 100644
index 00000000..0306ee1d
--- /dev/null
+++ b/src/rt/sync/fair_ticket_lock.cpp
@@ -0,0 +1,43 @@
+/*
+ * This works well as long as the number of contending threads
+ * is less than the number of processors. This is because of
+ * the fair locking scheme. If the thread that is next in line
+ * for acquiring the lock is not currently running, no other
+ * thread can acquire the lock. This is terrible for performance,
+ * and it seems that all fair locking schemes suffer from this
+ * behavior.
+ */
+
+// #define TRACE
+
+fair_ticket_lock::fair_ticket_lock() {
+    next_ticket = now_serving = 0;
+}
+
+fair_ticket_lock::~fair_ticket_lock() {
+
+}
+
+void fair_ticket_lock::lock() {
+    unsigned ticket = __sync_fetch_and_add(&next_ticket, 1);
+    while (now_serving != ticket) {
+        pause();
+    }
+#ifdef TRACE
+    printf("locked   nextTicket: %d nowServing: %d",
+            next_ticket, now_serving);
+#endif
+}
+
+void fair_ticket_lock::unlock() {
+    now_serving++;
+#ifdef TRACE
+    printf("unlocked nextTicket: %d nowServing: %d",
+            next_ticket, now_serving);
+#endif
+}
+
+void fair_ticket_lock::pause() {
+    asm volatile("pause\n" : : : "memory");
+}
+
diff --git a/src/rt/sync/fair_ticket_lock.h b/src/rt/sync/fair_ticket_lock.h
new file mode 100644
index 00000000..c34c9041
--- /dev/null
+++ b/src/rt/sync/fair_ticket_lock.h
@@ -0,0 +1,15 @@
+#ifndef FAIR_TICKET_LOCK_H
+#define FAIR_TICKET_LOCK_H
+
+class fair_ticket_lock {
+    unsigned next_ticket;
+    unsigned now_serving;
+    void pause();
+public:
+    fair_ticket_lock();
+    virtual ~fair_ticket_lock();
+    void lock();
+    void unlock();
+};
+
+#endif /* FAIR_TICKET_LOCK_H */
diff --git a/src/rt/sync/lock_free_queue.cpp b/src/rt/sync/lock_free_queue.cpp
new file mode 100644
index 00000000..9d1081de
--- /dev/null
+++ b/src/rt/sync/lock_free_queue.cpp
@@ -0,0 +1,37 @@
+/*
+ * Interrupt transparent queue, Schoen et. al, "On Interrupt-Transparent
+ * Synchronization in an Embedded Object-Oriented Operating System", 2000.
+ * enqueue() is allowed to interrupt enqueue() and dequeue(), however,
+ * dequeue() is not allowed to interrupt itself.
+ */
+
+#include "lock_free_queue.h"
+
+lock_free_queue::lock_free_queue() :
+    tail(this) {
+}
+
+void lock_free_queue::enqueue(lock_free_queue_node *item) {
+    item->next = (lock_free_queue_node *) 0;
+    lock_free_queue_node *last = tail;
+    tail = item;
+    while (last->next)
+        last = last->next;
+    last->next = item;
+}
+
+lock_free_queue_node *lockfree_queue::dequeue() {
+    lock_free_queue_node *item = next;
+    if (item && !(next = item->next)) {
+        tail = (lock_free_queue_node *) this;
+        if (item->next) {
+            lock_free_queue_node *lost = item->next;
+            lock_free_queue_node *help;
+            do {
+                help = lost->next;
+                enqueue(lost);
+            } while ((lost = help) != (lock_free_queue_node *) 0);
+        }
+    }
+    return item;
+}
diff --git a/src/rt/sync/lock_free_queue.h b/src/rt/sync/lock_free_queue.h
new file mode 100644
index 00000000..fba4aa9a
--- /dev/null
+++ b/src/rt/sync/lock_free_queue.h
@@ -0,0 +1,15 @@
+#ifndef LOCK_FREE_QUEUE_H
+#define LOCK_FREE_QUEUE_H
+
+class lock_free_queue_node {
+    lock_free_queue_node *next;
+};
+
+class lock_free_queue {
+public:
+    lock_free_queue();
+    void enqueue(lock_free_queue_node *item);
+    lock_free_queue_node *dequeue();
+};
+
+#endif /* LOCK_FREE_QUEUE_H */
diff --git a/src/rt/sync/spin_lock.cpp b/src/rt/sync/spin_lock.cpp
new file mode 100644
index 00000000..11a5cb20
--- /dev/null
+++ b/src/rt/sync/spin_lock.cpp
@@ -0,0 +1,47 @@
+/*
+ * Your average spin lock.
+ */
+
+#include "globals.h"
+
+// #define TRACE
+
+spin_lock::spin_lock() {
+    unlock();
+}
+
+spin_lock::~spin_lock() {
+}
+
+static inline unsigned xchg32(void *ptr, unsigned x) {
+    __asm__ __volatile__("xchgl %0,%1"
+                :"=r" ((unsigned) x)
+                :"m" (*(volatile unsigned *)ptr), "0" (x)
+                :"memory");
+    return x;
+}
+
+void spin_lock::lock() {
+    while (true) {
+        if (!xchg32(&ticket, 1)) {
+            return;
+        }
+        while (ticket) {
+            pause();
+        }
+    }
+#ifdef TRACE
+    printf("  lock: %d", ticket);
+#endif
+}
+
+void spin_lock::unlock() {
+    ticket = 0;
+#ifdef TRACE
+    printf("unlock:");
+#endif
+}
+
+void spin_lock::pause() {
+    asm volatile("pause\n" : : : "memory");
+}
diff --git a/src/rt/sync/spin_lock.h b/src/rt/sync/spin_lock.h
new file mode 100644
index 00000000..3684c23a
--- /dev/null
+++ b/src/rt/sync/spin_lock.h
@@ -0,0 +1,14 @@
+#ifndef UNFAIR_TICKET_LOCK_H
+#define UNFAIR_TICKET_LOCK_H
+
+class spin_lock {
+    unsigned ticket;
+    void pause();
+public:
+    spin_lock();
+    virtual ~spin_lock();
+    void lock();
+    void unlock();
+};
+
+#endif /* UNFAIR_TICKET_LOCK_H */
diff --git a/src/rt/uthash/uthash.h b/src/rt/uthash/uthash.h
new file mode 100644
index 00000000..28021b61
--- /dev/null
+++ b/src/rt/uthash/uthash.h
@@ -0,0 +1,766 @@
+/*
+Copyright (c) 2003-2009, Troy D. Hanson     http://uthash.sourceforge.net
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#ifndef UTHASH_H
+#define UTHASH_H 
+
+#include <string.h>   /* memcmp,strlen */
+#include <stddef.h>   /* ptrdiff_t */
+#include <inttypes.h> /* uint32_t etc */
+
+#define UTHASH_VERSION 1.6
+
+/* C++ requires extra stringent casting */
+#if defined __cplusplus
+#define TYPEOF(x) (typeof(x))
+#else
+#define TYPEOF(x)
+#endif
+
+
+#define uthash_fatal(msg) exit(-1)        /* fatal error (out of memory,etc) */
+#define uthash_bkt_malloc(sz) malloc(sz)  /* malloc fcn for UT_hash_bucket's */
+#define uthash_bkt_free(ptr) free(ptr)    /* free fcn for UT_hash_bucket's   */
+#define uthash_tbl_malloc(sz) malloc(sz)  /* malloc fcn for UT_hash_table    */
+#define uthash_tbl_free(ptr) free(ptr)    /* free fcn for UT_hash_table      */
+
+#define uthash_noexpand_fyi(tbl)          /* can be defined to log noexpand  */
+#define uthash_expand_fyi(tbl)            /* can be defined to log expands   */
+
+/* initial number of buckets */
+#define HASH_INITIAL_NUM_BUCKETS 32      /* initial number of buckets        */
+#define HASH_INITIAL_NUM_BUCKETS_LOG2 5  /* lg2 of initial number of buckets */
+#define HASH_BKT_CAPACITY_THRESH 10      /* expand when bucket count reaches */
+
+/* calculate the element whose hash handle address is hhe */
+#define ELMT_FROM_HH(tbl,hhp) ((void*)(((char*)hhp) - (tbl)->hho))
+
+#define HASH_FIND(hh,head,keyptr,keylen,out)                                   \
+do {                                                                           \
+  unsigned _hf_bkt,_hf_hashv;                                                  \
+  out=TYPEOF(out)head;                                                         \
+  if (head) {                                                                  \
+     HASH_FCN(keyptr,keylen, (head)->hh.tbl->num_buckets, _hf_hashv, _hf_bkt); \
+     HASH_FIND_IN_BKT((head)->hh.tbl, hh, (head)->hh.tbl->buckets[ _hf_bkt ],  \
+                      keyptr,keylen,out);                                      \
+  }                                                                            \
+} while (0)
+
+#define HASH_MAKE_TABLE(hh,head)                                               \
+do {                                                                           \
+  (head)->hh.tbl = (UT_hash_table*)uthash_tbl_malloc(                          \
+                  sizeof(UT_hash_table));                                      \
+  if (!((head)->hh.tbl))  { uthash_fatal( "out of memory"); }                  \
+  memset((head)->hh.tbl, 0, sizeof(UT_hash_table));                            \
+  (head)->hh.tbl->tail = &((head)->hh);                                        \
+  (head)->hh.tbl->num_buckets = HASH_INITIAL_NUM_BUCKETS;                      \
+  (head)->hh.tbl->log2_num_buckets = HASH_INITIAL_NUM_BUCKETS_LOG2;            \
+  (head)->hh.tbl->hho = (char*)(&(head)->hh) - (char*)(head);                  \
+  (head)->hh.tbl->buckets = (UT_hash_bucket*)uthash_bkt_malloc(                \
+          HASH_INITIAL_NUM_BUCKETS*sizeof(struct UT_hash_bucket));             \
+  if (! (head)->hh.tbl->buckets) { uthash_fatal( "out of memory"); }           \
+  memset((head)->hh.tbl->buckets, 0,                                           \
+          HASH_INITIAL_NUM_BUCKETS*sizeof(struct UT_hash_bucket));             \
+} while(0)
+
+#define HASH_ADD(hh,head,fieldname,keylen_in,add)                              \
+        HASH_ADD_KEYPTR(hh,head,&add->fieldname,keylen_in,add)
+ 
+#define HASH_ADD_KEYPTR(hh,head,keyptr,keylen_in,add)                          \
+do {                                                                           \
+ unsigned _ha_bkt;                                                             \
+ (add)->hh.next = NULL;                                                        \
+ (add)->hh.key = (char*)keyptr;                                                \
+ (add)->hh.keylen = keylen_in;                                                 \
+ if (!(head)) {                                                                \
+    head = (add);                                                              \
+    (head)->hh.prev = NULL;                                                    \
+    HASH_MAKE_TABLE(hh,head);                                                  \
+ } else {                                                                      \
+    (head)->hh.tbl->tail->next = (add);                                        \
+    (add)->hh.prev = ELMT_FROM_HH((head)->hh.tbl, (head)->hh.tbl->tail);       \
+    (head)->hh.tbl->tail = &((add)->hh);                                       \
+ }                                                                             \
+ (head)->hh.tbl->num_items++;                                                  \
+ (add)->hh.tbl = (head)->hh.tbl;                                               \
+ HASH_FCN(keyptr,keylen_in, (head)->hh.tbl->num_buckets,                       \
+         (add)->hh.hashv, _ha_bkt);                                            \
+ HASH_ADD_TO_BKT((head)->hh.tbl->buckets[_ha_bkt],&(add)->hh);                 \
+ HASH_EMIT_KEY(hh,head,keyptr,keylen_in);                                      \
+ HASH_FSCK(hh,head);                                                           \
+} while(0)
+
+#define HASH_TO_BKT( hashv, num_bkts, bkt )                                    \
+do {                                                                           \
+  bkt = ((hashv) & ((num_bkts) - 1));                                          \
+} while(0)
+
+/* delete "delptr" from the hash table.
+ * "the usual" patch-up process for the app-order doubly-linked-list.
+ * The use of _hd_hh_del below deserves special explanation.
+ * These used to be expressed using (delptr) but that led to a bug
+ * if someone used the same symbol for the head and deletee, like
+ *  HASH_DELETE(hh,users,users);
+ * We want that to work, but by changing the head (users) below
+ * we were forfeiting our ability to further refer to the deletee (users)
+ * in the patch-up process. Solution: use scratch space in the table to
+ * copy the deletee pointer, then the latter references are via that
+ * scratch pointer rather than through the repointed (users) symbol.
+ */
+#define HASH_DELETE(hh,head,delptr)                                            \
+do {                                                                           \
+    unsigned _hd_bkt;                                                          \
+    struct UT_hash_handle *_hd_hh_del;                                         \
+    if ( ((delptr)->hh.prev == NULL) && ((delptr)->hh.next == NULL) )  {       \
+        uthash_bkt_free((head)->hh.tbl->buckets );                             \
+        uthash_tbl_free((head)->hh.tbl);                                       \
+        head = NULL;                                                           \
+    } else {                                                                   \
+        _hd_hh_del = &((delptr)->hh);                                          \
+        if ((delptr) == ELMT_FROM_HH((head)->hh.tbl,(head)->hh.tbl->tail)) {   \
+            (head)->hh.tbl->tail =                                             \
+                (UT_hash_handle*)((char*)((delptr)->hh.prev) +                 \
+                (head)->hh.tbl->hho);                                          \
+        }                                                                      \
+        if ((delptr)->hh.prev) {                                               \
+            ((UT_hash_handle*)((char*)((delptr)->hh.prev) +                    \
+                    (head)->hh.tbl->hho))->next = (delptr)->hh.next;           \
+        } else {                                                               \
+            head = TYPEOF(head)((delptr)->hh.next);                            \
+        }                                                                      \
+        if (_hd_hh_del->next) {                                                \
+            ((UT_hash_handle*)((char*)_hd_hh_del->next +                       \
+                    (head)->hh.tbl->hho))->prev =                              \
+                    _hd_hh_del->prev;                                          \
+        }                                                                      \
+        HASH_TO_BKT( _hd_hh_del->hashv, (head)->hh.tbl->num_buckets, _hd_bkt); \
+        HASH_DEL_IN_BKT(hh,(head)->hh.tbl->buckets[_hd_bkt], _hd_hh_del);      \
+        (head)->hh.tbl->num_items--;                                           \
+    }                                                                          \
+    HASH_FSCK(hh,head);                                                        \
+} while (0)
+
+
+/* convenience forms of HASH_FIND/HASH_ADD/HASH_DEL */
+#define HASH_FIND_STR(head,findstr,out)                                        \
+    HASH_FIND(hh,head,findstr,strlen(findstr),out)
+#define HASH_ADD_STR(head,strfield,add)                                        \
+    HASH_ADD(hh,head,strfield,strlen(add->strfield),add)
+#define HASH_FIND_INT(head,findint,out)                                        \
+    HASH_FIND(hh,head,findint,sizeof(int),out)
+#define HASH_ADD_INT(head,intfield,add)                                        \
+    HASH_ADD(hh,head,intfield,sizeof(int),add)
+#define HASH_DEL(head,delptr)                                                  \
+    HASH_DELETE(hh,head,delptr)
+
+/* HASH_FSCK checks hash integrity on every add/delete when HASH_DEBUG is defined.
+ * This is for uthash developer only; it compiles away if HASH_DEBUG isn't defined.
+ */
+#ifdef HASH_DEBUG
+#define HASH_OOPS(...) do { fprintf(stderr,__VA_ARGS__); exit(-1); } while (0)
+#define HASH_FSCK(hh,head)                                                     \
+do {                                                                           \
+    unsigned _bkt_i;                                                           \
+    unsigned _count, _bkt_count;                                               \
+    char *_prev;                                                               \
+    struct UT_hash_handle *_thh;                                               \
+    if (head) {                                                                \
+        _count = 0;                                                            \
+        for( _bkt_i = 0; _bkt_i < (head)->hh.tbl->num_buckets; _bkt_i++) {     \
+            _bkt_count = 0;                                                    \
+            _thh = (head)->hh.tbl->buckets[_bkt_i].hh_head;                    \
+            _prev = NULL;                                                      \
+            while (_thh) {                                                     \
+               if (_prev != (char*)(_thh->hh_prev)) {                          \
+                   HASH_OOPS("invalid hh_prev %p, actual %p\n",                \
+                    _thh->hh_prev, _prev );                                    \
+               }                                                               \
+               _bkt_count++;                                                   \
+               _prev = (char*)(_thh);                                          \
+               _thh = _thh->hh_next;                                           \
+            }                                                                  \
+            _count += _bkt_count;                                              \
+            if ((head)->hh.tbl->buckets[_bkt_i].count !=  _bkt_count) {        \
+               HASH_OOPS("invalid bucket count %d, actual %d\n",               \
+                (head)->hh.tbl->buckets[_bkt_i].count, _bkt_count);            \
+            }                                                                  \
+        }                                                                      \
+        if (_count != (head)->hh.tbl->num_items) {                             \
+            HASH_OOPS("invalid hh item count %d, actual %d\n",                 \
+                (head)->hh.tbl->num_items, _count );                           \
+        }                                                                      \
+        /* traverse hh in app order; check next/prev integrity, count */       \
+        _count = 0;                                                            \
+        _prev = NULL;                                                          \
+        _thh =  &(head)->hh;                                                   \
+        while (_thh) {                                                         \
+           _count++;                                                           \
+           if (_prev !=(char*)(_thh->prev)) {                                  \
+              HASH_OOPS("invalid prev %p, actual %p\n",                        \
+                    _thh->prev, _prev );                                       \
+           }                                                                   \
+           _prev = (char*)ELMT_FROM_HH((head)->hh.tbl, _thh);                  \
+           _thh = ( _thh->next ?  (UT_hash_handle*)((char*)(_thh->next) +      \
+                                  (head)->hh.tbl->hho) : NULL );               \
+        }                                                                      \
+        if (_count != (head)->hh.tbl->num_items) {                             \
+            HASH_OOPS("invalid app item count %d, actual %d\n",                \
+                (head)->hh.tbl->num_items, _count );                           \
+        }                                                                      \
+    }                                                                          \
+} while (0)
+#else
+#define HASH_FSCK(hh,head) 
+#endif
+
+/* When compiled with -DHASH_EMIT_KEYS, length-prefixed keys are emitted to 
+ * the descriptor to which this macro is defined for tuning the hash function.
+ * The app can #include <unistd.h> to get the prototype for write(2). */
+#ifdef HASH_EMIT_KEYS
+#define HASH_EMIT_KEY(hh,head,keyptr,fieldlen)                                 \
+do {                                                                           \
+    unsigned _klen = fieldlen;                                                 \
+    write(HASH_EMIT_KEYS, &_klen, sizeof(_klen));                              \
+    write(HASH_EMIT_KEYS, keyptr, fieldlen);                                   \
+} while (0)
+#else 
+#define HASH_EMIT_KEY(hh,head,keyptr,fieldlen)                    
+#endif
+
+/* default to MurmurHash unless overridden e.g. DHASH_FUNCTION=HASH_SAX */
+#ifdef HASH_FUNCTION 
+#define HASH_FCN HASH_FUNCTION
+#else
+#define HASH_FCN HASH_MUR
+#endif
+
+/* The Bernstein hash function, used in Perl prior to v5.6 */
+#define HASH_BER(key,keylen,num_bkts,hashv,bkt)                                \
+do {                                                                           \
+  unsigned _hb_keylen=keylen;                                                  \
+  char *_hb_key=(char*)key;                                                    \
+  (hashv) = 0;                                                                 \
+  while (_hb_keylen--)  { (hashv) = ((hashv) * 33) + *_hb_key++; }             \
+  bkt = (hashv) & (num_bkts-1);                                                \
+} while (0)
+
+
+/* SAX/FNV/OAT/JEN hash functions are macro variants of those listed at 
+ * http://eternallyconfuzzled.com/tuts/algorithms/jsw_tut_hashing.aspx */
+#define HASH_SAX(key,keylen,num_bkts,hashv,bkt)                                \
+do {                                                                           \
+  unsigned _sx_i;                                                              \
+  char *_hs_key=(char*)key;                                                    \
+  hashv = 0;                                                                   \
+  for(_sx_i=0; _sx_i < keylen; _sx_i++)                                        \
+      hashv ^= (hashv << 5) + (hashv >> 2) + _hs_key[_sx_i];                   \
+  bkt = hashv & (num_bkts-1);                                                  \
+} while (0)
+
+#define HASH_FNV(key,keylen,num_bkts,hashv,bkt)                                \
+do {                                                                           \
+  unsigned _fn_i;                                                              \
+  char *_hf_key=(char*)key;                                                    \
+  hashv = 2166136261UL;                                                        \
+  for(_fn_i=0; _fn_i < keylen; _fn_i++)                                        \
+      hashv = (hashv * 16777619) ^ _hf_key[_fn_i];                             \
+  bkt = hashv & (num_bkts-1);                                                  \
+} while(0);
+ 
+#define HASH_OAT(key,keylen,num_bkts,hashv,bkt)                                \
+do {                                                                           \
+  unsigned _ho_i;                                                              \
+  char *_ho_key=(char*)key;                                                    \
+  hashv = 0;                                                                   \
+  for(_ho_i=0; _ho_i < keylen; _ho_i++) {                                      \
+      hashv += _ho_key[_ho_i];                                                 \
+      hashv += (hashv << 10);                                                  \
+      hashv ^= (hashv >> 6);                                                   \
+  }                                                                            \
+  hashv += (hashv << 3);                                                       \
+  hashv ^= (hashv >> 11);                                                      \
+  hashv += (hashv << 15);                                                      \
+  bkt = hashv & (num_bkts-1);                                                  \
+} while(0)
+
+#define HASH_JEN_MIX(a,b,c)                                                    \
+do {                                                                           \
+  a -= b; a -= c; a ^= ( c >> 13 );                                            \
+  b -= c; b -= a; b ^= ( a << 8 );                                             \
+  c -= a; c -= b; c ^= ( b >> 13 );                                            \
+  a -= b; a -= c; a ^= ( c >> 12 );                                            \
+  b -= c; b -= a; b ^= ( a << 16 );                                            \
+  c -= a; c -= b; c ^= ( b >> 5 );                                             \
+  a -= b; a -= c; a ^= ( c >> 3 );                                             \
+  b -= c; b -= a; b ^= ( a << 10 );                                            \
+  c -= a; c -= b; c ^= ( b >> 15 );                                            \
+} while (0)
+
+#define HASH_JEN(key,keylen,num_bkts,hashv,bkt)                                \
+do {                                                                           \
+  unsigned _hj_i,_hj_j,_hj_k;                                                  \
+  char *_hj_key=(char*)key;                                                    \
+  hashv = 0xfeedbeef;                                                          \
+  _hj_i = _hj_j = 0x9e3779b9;                                                  \
+  _hj_k = keylen;                                                              \
+  while (_hj_k >= 12) {                                                        \
+    _hj_i +=    (_hj_key[0] + ( (unsigned)_hj_key[1] << 8 )                    \
+        + ( (unsigned)_hj_key[2] << 16 )                                       \
+        + ( (unsigned)_hj_key[3] << 24 ) );                                    \
+    _hj_j +=    (_hj_key[4] + ( (unsigned)_hj_key[5] << 8 )                    \
+        + ( (unsigned)_hj_key[6] << 16 )                                       \
+        + ( (unsigned)_hj_key[7] << 24 ) );                                    \
+    hashv += (_hj_key[8] + ( (unsigned)_hj_key[9] << 8 )                       \
+        + ( (unsigned)_hj_key[10] << 16 )                                      \
+        + ( (unsigned)_hj_key[11] << 24 ) );                                   \
+                                                                               \
+     HASH_JEN_MIX(_hj_i, _hj_j, hashv);                                        \
+                                                                               \
+     _hj_key += 12;                                                            \
+     _hj_k -= 12;                                                              \
+  }                                                                            \
+  hashv += keylen;                                                             \
+  switch ( _hj_k ) {                                                           \
+     case 11: hashv += ( (unsigned)_hj_key[10] << 24 );                        \
+     case 10: hashv += ( (unsigned)_hj_key[9] << 16 );                         \
+     case 9:  hashv += ( (unsigned)_hj_key[8] << 8 );                          \
+     case 8:  _hj_j += ( (unsigned)_hj_key[7] << 24 );                         \
+     case 7:  _hj_j += ( (unsigned)_hj_key[6] << 16 );                         \
+     case 6:  _hj_j += ( (unsigned)_hj_key[5] << 8 );                          \
+     case 5:  _hj_j += _hj_key[4];                                             \
+     case 4:  _hj_i += ( (unsigned)_hj_key[3] << 24 );                         \
+     case 3:  _hj_i += ( (unsigned)_hj_key[2] << 16 );                         \
+     case 2:  _hj_i += ( (unsigned)_hj_key[1] << 8 );                          \
+     case 1:  _hj_i += _hj_key[0];                                             \
+  }                                                                            \
+  HASH_JEN_MIX(_hj_i, _hj_j, hashv);                                           \
+  bkt = hashv & (num_bkts-1);                                                  \
+} while(0)
+
+/* The Paul Hsieh hash function */
+#undef get16bits
+#if (defined(__GNUC__) && defined(__i386__)) || defined(__WATCOMC__)           \
+  || defined(_MSC_VER) || defined (__BORLANDC__) || defined (__TURBOC__)
+#define get16bits(d) (*((const uint16_t *) (d)))
+#endif
+
+#if !defined (get16bits)
+#define get16bits(d) ((((uint32_t)(((const uint8_t *)(d))[1])) << 8)\
+                       +(uint32_t)(((const uint8_t *)(d))[0]) )
+#endif
+#define HASH_SFH(key,keylen,num_bkts,hashv,bkt)                                \
+do {                                                                           \
+  char *_sfh_key=(char*)key;                                                   \
+  hashv = 0xcafebabe;                                                          \
+  uint32_t _sfh_tmp, _sfh_len = keylen;                                        \
+                                                                               \
+  int _sfh_rem = _sfh_len & 3;                                                 \
+  _sfh_len >>= 2;                                                              \
+                                                                               \
+  /* Main loop */                                                              \
+  for (;_sfh_len > 0; _sfh_len--) {                                            \
+    hashv    += get16bits (_sfh_key);                                          \
+    _sfh_tmp       = (get16bits (_sfh_key+2) << 11) ^ hashv;                   \
+    hashv     = (hashv << 16) ^ _sfh_tmp;                                      \
+    _sfh_key += 2*sizeof (uint16_t);                                           \
+    hashv    += hashv >> 11;                                                   \
+  }                                                                            \
+                                                                               \
+  /* Handle end cases */                                                       \
+  switch (_sfh_rem) {                                                          \
+    case 3: hashv += get16bits (_sfh_key);                                     \
+            hashv ^= hashv << 16;                                              \
+            hashv ^= _sfh_key[sizeof (uint16_t)] << 18;                        \
+            hashv += hashv >> 11;                                              \
+            break;                                                             \
+    case 2: hashv += get16bits (_sfh_key);                                     \
+            hashv ^= hashv << 11;                                              \
+            hashv += hashv >> 17;                                              \
+            break;                                                             \
+    case 1: hashv += *_sfh_key;                                                \
+            hashv ^= hashv << 10;                                              \
+            hashv += hashv >> 1;                                               \
+  }                                                                            \
+                                                                               \
+    /* Force "avalanching" of final 127 bits */                                \
+    hashv ^= hashv << 3;                                                       \
+    hashv += hashv >> 5;                                                       \
+    hashv ^= hashv << 4;                                                       \
+    hashv += hashv >> 17;                                                      \
+    hashv ^= hashv << 25;                                                      \
+    hashv += hashv >> 6;                                                       \
+    bkt = hashv & (num_bkts-1);                                                \
+} while(0);
+
+/* Austin Appleby's MurmurHash */
+#define HASH_MUR(key,keylen,num_bkts,hashv,bkt)                                \
+do {                                                                           \
+  const unsigned int _mur_m = 0x5bd1e995;                                      \
+  const int _mur_r = 24;                                                       \
+  hashv = 0xcafebabe ^ keylen;                                                 \
+  char *_mur_key = (char *)key;                                                \
+  uint32_t _mur_tmp, _mur_len = keylen;                                        \
+                                                                               \
+  for (;_mur_len >= 4; _mur_len-=4) {                                          \
+    _mur_tmp = *(uint32_t *)_mur_key;                                          \
+    _mur_tmp *= _mur_m;                                                        \
+    _mur_tmp ^= _mur_tmp >> _mur_r;                                            \
+    _mur_tmp *= _mur_m;                                                        \
+    hashv *= _mur_m;                                                           \
+    hashv ^= _mur_tmp;                                                         \
+    _mur_key += 4;                                                             \
+  }                                                                            \
+                                                                               \
+  switch(_mur_len)                                                             \
+  {                                                                            \
+    case 3: hashv ^= _mur_key[2] << 16;                                        \
+    case 2: hashv ^= _mur_key[1] << 8;                                         \
+    case 1: hashv ^= _mur_key[0];                                              \
+            hashv *= _mur_m;                                                   \
+  };                                                                           \
+                                                                               \
+  hashv ^= hashv >> 13;                                                        \
+  hashv *= _mur_m;                                                             \
+  hashv ^= hashv >> 15;                                                        \
+                                                                               \
+  bkt = hashv & (num_bkts-1);                                                  \
+} while(0)
+
+/* key comparison function; return 0 if keys equal */
+#define HASH_KEYCMP(a,b,len) memcmp(a,b,len) 
+
+/* iterate over items in a known bucket to find desired item */
+#define HASH_FIND_IN_BKT(tbl,hh,head,keyptr,keylen_in,out)                     \
+out = TYPEOF(out)((head.hh_head) ? ELMT_FROM_HH(tbl,head.hh_head) : NULL);     \
+while (out) {                                                                  \
+    if (out->hh.keylen == keylen_in) {                                         \
+        if ((HASH_KEYCMP(out->hh.key,keyptr,keylen_in)) == 0) break;           \
+    }                                                                          \
+    out= TYPEOF(out)((out->hh.hh_next) ?                                       \
+                     ELMT_FROM_HH(tbl,out->hh.hh_next) : NULL);                \
+}
+
+/* add an item to a bucket  */
+#define HASH_ADD_TO_BKT(head,addhh)                                            \
+do {                                                                           \
+ head.count++;                                                                 \
+ (addhh)->hh_next = head.hh_head;                                              \
+ (addhh)->hh_prev = NULL;                                                      \
+ if (head.hh_head) { (head).hh_head->hh_prev = (addhh); }                      \
+ (head).hh_head=addhh;                                                         \
+ if (head.count >= ((head.expand_mult+1) * HASH_BKT_CAPACITY_THRESH)           \
+     && (addhh)->tbl->noexpand != 1) {                                         \
+       HASH_EXPAND_BUCKETS((addhh)->tbl);                                      \
+ }                                                                             \
+} while(0)
+
+/* remove an item from a given bucket */
+#define HASH_DEL_IN_BKT(hh,head,hh_del)                                        \
+    (head).count--;                                                            \
+    if ((head).hh_head == hh_del) {                                            \
+      (head).hh_head = hh_del->hh_next;                                        \
+    }                                                                          \
+    if (hh_del->hh_prev) {                                                     \
+        hh_del->hh_prev->hh_next = hh_del->hh_next;                            \
+    }                                                                          \
+    if (hh_del->hh_next) {                                                     \
+        hh_del->hh_next->hh_prev = hh_del->hh_prev;                            \
+    }                                                                
+
+/* Bucket expansion has the effect of doubling the number of buckets
+ * and redistributing the items into the new buckets. Ideally the
+ * items will distribute more or less evenly into the new buckets
+ * (the extent to which this is true is a measure of the quality of
+ * the hash function as it applies to the key domain). 
+ * 
+ * With the items distributed into more buckets, the chain length
+ * (item count) in each bucket is reduced. Thus by expanding buckets
+ * the hash keeps a bound on the chain length. This bounded chain 
+ * length is the essence of how a hash provides constant time lookup.
+ * 
+ * The calculation of tbl->ideal_chain_maxlen below deserves some
+ * explanation. First, keep in mind that we're calculating the ideal
+ * maximum chain length based on the *new* (doubled) bucket count.
+ * In fractions this is just n/b (n=number of items,b=new num buckets).
+ * Since the ideal chain length is an integer, we want to calculate 
+ * ceil(n/b). We don't depend on floating point arithmetic in this
+ * hash, so to calculate ceil(n/b) with integers we could write
+ * 
+ *      ceil(n/b) = (n/b) + ((n%b)?1:0)
+ * 
+ * and in fact a previous version of this hash did just that.
+ * But now we have improved things a bit by recognizing that b is
+ * always a power of two. We keep its base 2 log handy (call it lb),
+ * so now we can write this with a bit shift and logical AND:
+ * 
+ *      ceil(n/b) = (n>>lb) + ( (n & (b-1)) ? 1:0)
+ * 
+ */
+#define HASH_EXPAND_BUCKETS(tbl)                                               \
+do {                                                                           \
+    unsigned _he_bkt;                                                          \
+    unsigned _he_bkt_i;                                                        \
+    struct UT_hash_handle *_he_thh, *_he_hh_nxt;                               \
+    UT_hash_bucket *_he_new_buckets, *_he_newbkt;                              \
+    _he_new_buckets = (UT_hash_bucket*)uthash_bkt_malloc(                      \
+             2 * tbl->num_buckets * sizeof(struct UT_hash_bucket));            \
+    if (!_he_new_buckets) { uthash_fatal( "out of memory"); }                  \
+    memset(_he_new_buckets, 0,                                                 \
+            2 * tbl->num_buckets * sizeof(struct UT_hash_bucket));             \
+    tbl->ideal_chain_maxlen =                                                  \
+       (tbl->num_items >> (tbl->log2_num_buckets+1)) +                         \
+       ((tbl->num_items & ((tbl->num_buckets*2)-1)) ? 1 : 0);                  \
+    tbl->nonideal_items = 0;                                                   \
+    for(_he_bkt_i = 0; _he_bkt_i < tbl->num_buckets; _he_bkt_i++)              \
+    {                                                                          \
+        _he_thh = tbl->buckets[ _he_bkt_i ].hh_head;                           \
+        while (_he_thh) {                                                      \
+           _he_hh_nxt = _he_thh->hh_next;                                      \
+           HASH_TO_BKT( _he_thh->hashv, tbl->num_buckets*2, _he_bkt);          \
+           _he_newbkt = &(_he_new_buckets[ _he_bkt ]);                         \
+           if (++(_he_newbkt->count) > tbl->ideal_chain_maxlen) {              \
+             tbl->nonideal_items++;                                            \
+             _he_newbkt->expand_mult = _he_newbkt->count /                     \
+                                        tbl->ideal_chain_maxlen;               \
+           }                                                                   \
+           _he_thh->hh_prev = NULL;                                            \
+           _he_thh->hh_next = _he_newbkt->hh_head;                             \
+           if (_he_newbkt->hh_head) _he_newbkt->hh_head->hh_prev =             \
+                _he_thh;                                                       \
+           _he_newbkt->hh_head = _he_thh;                                      \
+           _he_thh = _he_hh_nxt;                                               \
+        }                                                                      \
+    }                                                                          \
+    tbl->num_buckets *= 2;                                                     \
+    tbl->log2_num_buckets++;                                                   \
+    uthash_bkt_free( tbl->buckets );                                           \
+    tbl->buckets = _he_new_buckets;                                            \
+    tbl->ineff_expands = (tbl->nonideal_items > (tbl->num_items >> 1)) ?       \
+        (tbl->ineff_expands+1) : 0;                                            \
+    if (tbl->ineff_expands > 1) {                                              \
+        tbl->noexpand=1;                                                       \
+        uthash_noexpand_fyi(tbl);                                              \
+    }                                                                          \
+    uthash_expand_fyi(tbl);                                                    \
+} while(0)
+
+
+/* This is an adaptation of Simon Tatham's O(n log(n)) mergesort */
+/* Note that HASH_SORT assumes the hash handle name to be hh. 
+ * HASH_SRT was added to allow the hash handle name to be passed in. */
+#define HASH_SORT(head,cmpfcn) HASH_SRT(hh,head,cmpfcn)
+#define HASH_SRT(hh,head,cmpfcn)                                               \
+do {                                                                           \
+  unsigned _hs_i;                                                              \
+  unsigned _hs_looping,_hs_nmerges,_hs_insize,_hs_psize,_hs_qsize;             \
+  struct UT_hash_handle *_hs_p, *_hs_q, *_hs_e, *_hs_list, *_hs_tail;          \
+  if (head) {                                                                  \
+      _hs_insize = 1;                                                          \
+      _hs_looping = 1;                                                         \
+      _hs_list = &((head)->hh);                                                \
+      while (_hs_looping) {                                                    \
+          _hs_p = _hs_list;                                                    \
+          _hs_list = NULL;                                                     \
+          _hs_tail = NULL;                                                     \
+          _hs_nmerges = 0;                                                     \
+          while (_hs_p) {                                                      \
+              _hs_nmerges++;                                                   \
+              _hs_q = _hs_p;                                                   \
+              _hs_psize = 0;                                                   \
+              for ( _hs_i = 0; _hs_i  < _hs_insize; _hs_i++ ) {                \
+                  _hs_psize++;                                                 \
+                  _hs_q = (UT_hash_handle*)((_hs_q->next) ?                    \
+                          ((void*)((char*)(_hs_q->next) +                      \
+                          (head)->hh.tbl->hho)) : NULL);                       \
+                  if (! (_hs_q) ) break;                                       \
+              }                                                                \
+              _hs_qsize = _hs_insize;                                          \
+              while ((_hs_psize > 0) || ((_hs_qsize > 0) && _hs_q )) {         \
+                  if (_hs_psize == 0) {                                        \
+                      _hs_e = _hs_q;                                           \
+                      _hs_q = (UT_hash_handle*)((_hs_q->next) ?                \
+                              ((void*)((char*)(_hs_q->next) +                  \
+                              (head)->hh.tbl->hho)) : NULL);                   \
+                      _hs_qsize--;                                             \
+                  } else if ( (_hs_qsize == 0) || !(_hs_q) ) {                 \
+                      _hs_e = _hs_p;                                           \
+                      _hs_p = (UT_hash_handle*)((_hs_p->next) ?                \
+                              ((void*)((char*)(_hs_p->next) +                  \
+                              (head)->hh.tbl->hho)) : NULL);                   \
+                      _hs_psize--;                                             \
+                  } else if ((                                                 \
+                      cmpfcn(TYPEOF(head)(ELMT_FROM_HH((head)->hh.tbl,_hs_p)), \
+                            TYPEOF(head)(ELMT_FROM_HH((head)->hh.tbl,_hs_q)))  \
+                             ) <= 0) {                                         \
+                      _hs_e = _hs_p;                                           \
+                      _hs_p = (UT_hash_handle*)((_hs_p->next) ?                \
+                              ((void*)((char*)(_hs_p->next) +                  \
+                              (head)->hh.tbl->hho)) : NULL);                   \
+                      _hs_psize--;                                             \
+                  } else {                                                     \
+                      _hs_e = _hs_q;                                           \
+                      _hs_q = (UT_hash_handle*)((_hs_q->next) ?                \
+                              ((void*)((char*)(_hs_q->next) +                  \
+                              (head)->hh.tbl->hho)) : NULL);                   \
+                      _hs_qsize--;                                             \
+                  }                                                            \
+                  if ( _hs_tail ) {                                            \
+                      _hs_tail->next = ((_hs_e) ?                              \
+                            ELMT_FROM_HH((head)->hh.tbl,_hs_e) : NULL);        \
+                  } else {                                                     \
+                      _hs_list = _hs_e;                                        \
+                  }                                                            \
+                  _hs_e->prev = ((_hs_tail) ?                                  \
+                     ELMT_FROM_HH((head)->hh.tbl,_hs_tail) : NULL);            \
+                  _hs_tail = _hs_e;                                            \
+              }                                                                \
+              _hs_p = _hs_q;                                                   \
+          }                                                                    \
+          _hs_tail->next = NULL;                                               \
+          if ( _hs_nmerges <= 1 ) {                                            \
+              _hs_looping=0;                                                   \
+              (head)->hh.tbl->tail = _hs_tail;                                 \
+              (head) = TYPEOF(head)ELMT_FROM_HH((head)->hh.tbl, _hs_list);     \
+          }                                                                    \
+          _hs_insize *= 2;                                                     \
+      }                                                                        \
+      HASH_FSCK(hh,head);                                                      \
+ }                                                                             \
+} while (0)
+
+/* This function selects items from one hash into another hash. 
+ * The end result is that the selected items have dual presence 
+ * in both hashes. There is no copy of the items made; rather 
+ * they are added into the new hash through a secondary hash 
+ * hash handle that must be present in the structure. */
+#define HASH_SELECT(hh_dst, dst, hh_src, src, cond)                            \
+do {                                                                           \
+  unsigned _src_bkt, _dst_bkt;                                                 \
+  void *_last_elt=NULL, *_elt;                                                 \
+  UT_hash_handle *_src_hh, *_dst_hh, *_last_elt_hh=NULL;                       \
+  ptrdiff_t _dst_hho = ((char*)(&(dst)->hh_dst) - (char*)(dst));               \
+  if (src) {                                                                   \
+    for(_src_bkt=0; _src_bkt < (src)->hh_src.tbl->num_buckets; _src_bkt++) {   \
+      for(_src_hh = (src)->hh_src.tbl->buckets[_src_bkt].hh_head;              \
+          _src_hh;                                                             \
+          _src_hh = _src_hh->hh_next) {                                        \
+          _elt = ELMT_FROM_HH((src)->hh_src.tbl, _src_hh);                     \
+          if (cond(_elt)) {                                                    \
+            _dst_hh = (UT_hash_handle*)(((char*)_elt) + _dst_hho);             \
+            _dst_hh->key = _src_hh->key;                                       \
+            _dst_hh->keylen = _src_hh->keylen;                                 \
+            _dst_hh->hashv = _src_hh->hashv;                                   \
+            _dst_hh->prev = _last_elt;                                         \
+            _dst_hh->next = NULL;                                              \
+            if (_last_elt_hh) { _last_elt_hh->next = _elt; }                   \
+            if (!dst) {                                                        \
+              dst = TYPEOF(dst)_elt;                                           \
+              HASH_MAKE_TABLE(hh_dst,dst);                                     \
+            } else {                                                           \
+              _dst_hh->tbl = (dst)->hh_dst.tbl;                                \
+            }                                                                  \
+            HASH_TO_BKT(_dst_hh->hashv, _dst_hh->tbl->num_buckets, _dst_bkt);  \
+            HASH_ADD_TO_BKT(_dst_hh->tbl->buckets[_dst_bkt],_dst_hh);          \
+            (dst)->hh_dst.tbl->num_items++;                                    \
+            _last_elt = _elt;                                                  \
+            _last_elt_hh = _dst_hh;                                            \
+          }                                                                    \
+      }                                                                        \
+    }                                                                          \
+  }                                                                            \
+  HASH_FSCK(hh_dst,dst);                                                       \
+} while (0)
+
+#define HASH_CLEAR(hh,head)                                                    \
+do {                                                                           \
+  if (head) {                                                                  \
+    uthash_bkt_free((head)->hh.tbl->buckets );                                 \
+    uthash_tbl_free((head)->hh.tbl);                                           \
+    (head)=NULL;                                                               \
+  }                                                                            \
+} while(0)
+
+/* obtain a count of items in the hash */
+#define HASH_COUNT(head) HASH_CNT(hh,head) 
+#define HASH_CNT(hh,head) (head?(head->hh.tbl->num_items):0)
+
+typedef struct UT_hash_bucket {
+   struct UT_hash_handle *hh_head;
+   unsigned count;
+
+   /* expand_mult is normally set to 0. In this situation, the max chain length
+    * threshold is enforced at its default value, HASH_BKT_CAPACITY_THRESH. (If
+    * the bucket's chain exceeds this length, bucket expansion is triggered). 
+    * However, setting expand_mult to a non-zero value delays bucket expansion
+    * (that would be triggered by additions to this particular bucket)
+    * until its chain length reaches a *multiple* of HASH_BKT_CAPACITY_THRESH.
+    * (The multiplier is simply expand_mult+1). The whole idea of this
+    * multiplier is to reduce bucket expansions, since they are expensive, in
+    * situations where we know that a particular bucket tends to be overused.
+    * It is better to let its chain length grow to a longer yet-still-bounded
+    * value, than to do an O(n) bucket expansion too often. 
+    */
+   unsigned expand_mult;
+
+} UT_hash_bucket;
+
+typedef struct UT_hash_table {
+   UT_hash_bucket *buckets;
+   unsigned num_buckets, log2_num_buckets;
+   unsigned num_items;
+   struct UT_hash_handle *tail; /* tail hh in app order, for fast append    */
+   ptrdiff_t hho; /* hash handle offset (byte pos of hash handle in element */
+
+   /* in an ideal situation (all buckets used equally), no bucket would have
+    * more than ceil(#items/#buckets) items. that's the ideal chain length. */
+   unsigned ideal_chain_maxlen;
+
+   /* nonideal_items is the number of items in the hash whose chain position
+    * exceeds the ideal chain maxlen. these items pay the penalty for an uneven
+    * hash distribution; reaching them in a chain traversal takes >ideal steps */
+   unsigned nonideal_items;
+
+   /* ineffective expands occur when a bucket doubling was performed, but 
+    * afterward, more than half the items in the hash had nonideal chain
+    * positions. If this happens on two consecutive expansions we inhibit any
+    * further expansion, as it's not helping; this happens when the hash
+    * function isn't a good fit for the key domain. When expansion is inhibited
+    * the hash will still work, albeit no longer in constant time. */
+   unsigned ineff_expands, noexpand;
+
+
+} UT_hash_table;
+
+
+typedef struct UT_hash_handle {
+   struct UT_hash_table *tbl;
+   void *prev;                       /* prev element in app order      */
+   void *next;                       /* next element in app order      */
+   struct UT_hash_handle *hh_prev;   /* previous hh in bucket order    */
+   struct UT_hash_handle *hh_next;   /* next hh in bucket order        */
+   void *key;                        /* ptr to enclosing struct's key  */
+   unsigned keylen;                  /* enclosing struct's key len     */
+   unsigned hashv;                   /* result of hash-fcn(key)        */
+} UT_hash_handle;
+
+#endif /* UTHASH_H */
diff --git a/src/rt/uthash/utlist.h b/src/rt/uthash/utlist.h
new file mode 100644
index 00000000..a33615e1
--- /dev/null
+++ b/src/rt/uthash/utlist.h
@@ -0,0 +1,280 @@
+/*
+Copyright (c) 2007-2009, Troy D. Hanson
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#ifndef UTLIST_H
+#define UTLIST_H
+
+#define UTLIST_VERSION 1.0
+
+/* C++ requires extra stringent casting */
+#if defined __cplusplus
+#define LTYPEOF(x) (typeof(x))
+#else
+#define LTYPEOF(x)
+#endif
+/* 
+ * This file contains macros to manipulate singly and doubly-linked lists.
+ *
+ * 1. LL_ macros:  singly-linked lists.
+ * 2. DL_ macros:  doubly-linked lists.
+ * 3. CDL_ macros: circular doubly-linked lists.
+ *
+ * To use singly-linked lists, your structure must have a "next" pointer.
+ * To use doubly-linked lists, your structure must "prev" and "next" pointers.
+ * Either way, the pointer to the head of the list must be initialized to NULL.
+ * 
+ * ----------------.EXAMPLE -------------------------
+ * struct item {
+ *      int id;
+ *      struct item *prev, *next;
+ * }
+ *
+ * struct item *list = NULL:
+ *
+ * int main() {
+ *      struct item *item;
+ *      ... allocate and populate item ...
+ *      DL_APPEND(list, item);
+ * }
+ * --------------------------------------------------
+ *
+ * For doubly-linked lists, the append and delete macros are O(1)
+ * For singly-linked lists, append and delete are O(n) but prepend is O(1)
+ * The sort macro is O(n log(n)) for all types of single/double/circular lists.
+ */
+
+/******************************************************************************
+ * The SORT macros                                                            *
+ *****************************************************************************/
+#define LL_SORT(l,cmp)                                                           \
+ LISTSORT(l,0,0,FIELD_OFFSET(l,next),cmp)
+#define DL_SORT(l,cmp)                                                           \
+ LISTSORT(l,0,FIELD_OFFSET(l,prev),FIELD_OFFSET(l,next),cmp)
+#define CDL_SORT(l,cmp)                                                          \
+ LISTSORT(l,1,FIELD_OFFSET(l,prev),FIELD_OFFSET(l,next),cmp)
+
+/* The macros can't assume or cast to the caller's list element type. So we use
+ * a couple tricks when we need to deal with those element's prev/next pointers.
+ * Basically we use char pointer arithmetic to get those field offsets. */
+#define FIELD_OFFSET(ptr,field) ((char*)&((ptr)->field) - (char*)(ptr))
+#define LNEXT(e,no) (*(char**)(((char*)e) + no))
+#define LPREV(e,po) (*(char**)(((char*)e) + po))
+/******************************************************************************
+ * The LISTSORT macro is an adaptation of Simon Tatham's O(n log(n)) mergesort*
+ * Unwieldy variable names used here to avoid shadowing passed-in variables.  *
+ *****************************************************************************/
+#define LISTSORT(list, is_circular, po, no, cmp)                                 \
+do {                                                                             \
+  void *_ls_p, *_ls_q, *_ls_e, *_ls_tail, *_ls_oldhead;                          \
+  int _ls_insize, _ls_nmerges, _ls_psize, _ls_qsize, _ls_i, _ls_looping;         \
+  int _ls_is_double = (po==0) ? 0 : 1;                                           \
+  if (list) {                                                                    \
+    _ls_insize = 1;                                                              \
+    _ls_looping = 1;                                                             \
+    while (_ls_looping) {                                                        \
+      _ls_p = list;                                                              \
+      _ls_oldhead = list;                                                        \
+      list = NULL;                                                               \
+      _ls_tail = NULL;                                                           \
+      _ls_nmerges = 0;                                                           \
+      while (_ls_p) {                                                            \
+        _ls_nmerges++;                                                           \
+        _ls_q = _ls_p;                                                           \
+        _ls_psize = 0;                                                           \
+        for (_ls_i = 0; _ls_i < _ls_insize; _ls_i++) {                           \
+          _ls_psize++;                                                           \
+          if (is_circular)  {                                                    \
+            _ls_q = ((LNEXT(_ls_q,no) == _ls_oldhead) ? NULL : LNEXT(_ls_q,no)); \
+          } else  {                                                              \
+            _ls_q = LNEXT(_ls_q,no);                                             \
+          }                                                                      \
+          if (!_ls_q) break;                                                     \
+        }                                                                        \
+        _ls_qsize = _ls_insize;                                                  \
+        while (_ls_psize > 0 || (_ls_qsize > 0 && _ls_q)) {                      \
+          if (_ls_psize == 0) {                                                  \
+            _ls_e = _ls_q; _ls_q = LNEXT(_ls_q,no); _ls_qsize--;                 \
+            if (is_circular && _ls_q == _ls_oldhead) { _ls_q = NULL; }           \
+          } else if (_ls_qsize == 0 || !_ls_q) {                                 \
+            _ls_e = _ls_p; _ls_p = LNEXT(_ls_p,no); _ls_psize--;                 \
+            if (is_circular && (_ls_p == _ls_oldhead)) { _ls_p = NULL; }         \
+          } else if (cmp(LTYPEOF(list)_ls_p,LTYPEOF(list)_ls_q) <= 0) {          \
+            _ls_e = _ls_p; _ls_p = LNEXT(_ls_p,no); _ls_psize--;                 \
+            if (is_circular && (_ls_p == _ls_oldhead)) { _ls_p = NULL; }         \
+          } else {                                                               \
+            _ls_e = _ls_q; _ls_q = LNEXT(_ls_q,no); _ls_qsize--;                 \
+            if (is_circular && (_ls_q == _ls_oldhead)) { _ls_q = NULL; }         \
+          }                                                                      \
+          if (_ls_tail) {                                                        \
+            LNEXT(_ls_tail,no) = (char*)_ls_e;                                   \
+          } else {                                                               \
+            list = LTYPEOF(list)_ls_e;                                           \
+          }                                                                      \
+          if (_ls_is_double) {                                                   \
+            LPREV(_ls_e,po) = (char*)_ls_tail;                                   \
+          }                                                                      \
+          _ls_tail = _ls_e;                                                      \
+        }                                                                        \
+        _ls_p = _ls_q;                                                           \
+      }                                                                          \
+      if (is_circular) {                                                         \
+        LNEXT(_ls_tail,no) = (char*)list;                                        \
+        if (_ls_is_double) {                                                     \
+          LPREV(list,po) = (char*)_ls_tail;                                      \
+        }                                                                        \
+      } else  {                                                                  \
+        LNEXT(_ls_tail,no) = NULL;                                               \
+      }                                                                          \
+      if (_ls_nmerges <= 1) {                                                    \
+        _ls_looping=0;                                                           \
+      }                                                                          \
+      _ls_insize *= 2;                                                           \
+    }                                                                            \
+  }                                                                              \
+} while (0)
+
+/******************************************************************************
+ * singly linked list macros (non-circular)                                   *
+ *****************************************************************************/
+#define LL_PREPEND(head,add)                                                     \
+do {                                                                             \
+  (add)->next = head;                                                            \
+  head = add;                                                                    \
+} while (0)
+
+#define LL_APPEND(head,add)                                                      \
+do {                                                                             \
+  (add)->next=NULL;                                                              \
+  if (head) {                                                                    \
+    char *_lla_el = (char*)(head);                                               \
+    unsigned _lla_no = FIELD_OFFSET(head,next);                                  \
+    while (LNEXT(_lla_el,_lla_no)) { _lla_el = LNEXT(_lla_el,_lla_no); }         \
+    LNEXT(_lla_el,_lla_no)=(char*)(add);                                         \
+  } else {                                                                       \
+    (head)=(add);                                                                \
+  }                                                                              \
+} while (0)
+
+#define LL_DELETE(head,del)                                                      \
+do {                                                                             \
+  if ((head) == (del)) {                                                         \
+    (head)=(head)->next;                                                         \
+  } else {                                                                       \
+    char *_lld_el = (char*)(head);                                               \
+    unsigned _lld_no = FIELD_OFFSET(head,next);                                  \
+    while (LNEXT(_lld_el,_lld_no) && (LNEXT(_lld_el,_lld_no) != (char*)(del))) { \
+      _lld_el = LNEXT(_lld_el,_lld_no);                                          \
+    }                                                                            \
+    if (LNEXT(_lld_el,_lld_no)) {                                                \
+      LNEXT(_lld_el,_lld_no) = (char*)((del)->next);                             \
+    }                                                                            \
+  }                                                                              \
+} while (0)
+
+#define LL_FOREACH(head,el)                                                      \
+    for(el=head;el;el=el->next)
+
+/******************************************************************************
+ * doubly linked list macros (non-circular)                                   *
+ *****************************************************************************/
+#define DL_PREPEND(head,add)                                                     \
+do {                                                                             \
+ (add)->next = head;                                                             \
+ if (head) {                                                                     \
+   (add)->prev = (head)->prev;                                                   \
+   (head)->prev = (add);                                                         \
+ } else {                                                                        \
+   (add)->prev = (add);                                                          \
+ }                                                                               \
+ (head) = (add);                                                                 \
+} while (0)
+
+#define DL_APPEND(head,add)                                                      \
+do {                                                                             \
+  if (head) {                                                                    \
+      (add)->prev = (head)->prev;                                                \
+      (head)->prev->next = (add);                                                \
+      (head)->prev = (add);                                                      \
+      (add)->next = NULL;                                                        \
+  } else {                                                                       \
+      (head)=(add);                                                              \
+      (head)->prev = (head);                                                     \
+      (head)->next = NULL;                                                       \
+  }                                                                              \
+} while (0);
+
+#define DL_DELETE(head,del)                                                      \
+do {                                                                             \
+  if ((del)->prev == (del)) {                                                    \
+      (head)=NULL;                                                               \
+  } else if ((del)==(head)) {                                                    \
+      (del)->next->prev = (del)->prev;                                           \
+      (head) = (del)->next;                                                      \
+  } else {                                                                       \
+      (del)->prev->next = (del)->next;                                           \
+      if ((del)->next) {                                                         \
+          (del)->next->prev = (del)->prev;                                       \
+      } else {                                                                   \
+          (head)->prev = (del)->prev;                                            \
+      }                                                                          \
+  }                                                                              \
+} while (0);
+
+
+#define DL_FOREACH(head,el)                                                      \
+    for(el=head;el;el=el->next)
+
+/******************************************************************************
+ * circular doubly linked list macros                                         *
+ *****************************************************************************/
+#define CDL_PREPEND(head,add)                                                    \
+do {                                                                             \
+ if (head) {                                                                     \
+   (add)->prev = (head)->prev;                                                   \
+   (add)->next = (head);                                                         \
+   (head)->prev = (add);                                                         \
+   (add)->prev->next = (add);                                                    \
+ } else {                                                                        \
+   (add)->prev = (add);                                                          \
+   (add)->next = (add);                                                          \
+ }                                                                               \
+(head)=(add);                                                                    \
+} while (0)
+
+#define CDL_DELETE(head,del)                                                     \
+do {                                                                             \
+  if ( ((head)==(del)) && ((head)->next == (head))) {                            \
+      (head) = 0L;                                                               \
+  } else {                                                                       \
+     (del)->next->prev = (del)->prev;                                            \
+     (del)->prev->next = (del)->next;                                            \
+     if ((del) == (head)) (head)=(del)->next;                                    \
+  }                                                                              \
+} while (0);
+
+#define CDL_FOREACH(head,el)                                                     \
+    for(el=head;el;el= (el->next==head ? 0L : el->next)) 
+
+
+#endif /* UTLIST_H */
+
diff --git a/src/rt/util/array_list.h b/src/rt/util/array_list.h
new file mode 100644
index 00000000..0d112575
--- /dev/null
+++ b/src/rt/util/array_list.h
@@ -0,0 +1,69 @@
+#ifndef ARRAY_LIST_H
+#define ARRAY_LIST_H
+
+/**
+ * A simple, resizable array list.
+ */
+template<typename T> class array_list {
+    static const size_t INITIAL_CAPACITY = 8;
+    size_t _size;
+    T * _data;
+    size_t _capacity;
+public:
+    array_list();
+    ~array_list();
+    size_t size();
+    void append(T value);
+    T replace(T old_value, T new_value);
+    size_t index_of(T value);
+    T & operator[](size_t index);
+};
+
+template<typename T> array_list<T>::array_list() {
+    _capacity = INITIAL_CAPACITY;
+    _data = (T *) malloc(sizeof(T) * _capacity);
+}
+
+template<typename T> array_list<T>::~array_list() {
+    delete _data;
+}
+
+template<typename T> size_t array_list<T>::size() {
+    return _size;
+}
+
+template<typename T> void array_list<T>::append(T value) {
+    if (_size == _capacity) {
+        _capacity = _capacity * 2;
+        _data = (T *) realloc(_data, _capacity * sizeof(T));
+    }
+    _data[_size++] = value;
+}
+
+/**
+ * Replaces the old_value in the list with the new_value.
+ * Returns the old_value if the replacement succeeded, or NULL otherwise.
+ */
+template<typename T> T array_list<T>::replace(T old_value, T new_value) {
+    int index = index_of(old_value);
+    if (index < 0) {
+        return NULL;
+    }
+    _data[index] = new_value;
+    return old_value;
+}
+
+template<typename T> size_t array_list<T>::index_of(T value) {
+    for (size_t i = 0; i < _size; i++) {
+        if (_data[i] == value) {
+            return i;
+        }
+    }
+    return -1;
+}
+
+template<typename T> T & array_list<T>::operator[](size_t index) {
+    return _data[index];
+}
+
+#endif /* ARRAY_LIST_H */
diff --git a/src/rt/valgrind.h b/src/rt/valgrind.h
new file mode 100644
index 00000000..530fa184
--- /dev/null
+++ b/src/rt/valgrind.h
@@ -0,0 +1,3926 @@
+/* -*- c -*-
+   ----------------------------------------------------------------
+
+   Notice that the following BSD-style license applies to this one
+   file (valgrind.h) only.  The rest of Valgrind is licensed under the
+   terms of the GNU General Public License, version 2, unless
+   otherwise indicated.  See the COPYING file in the source
+   distribution for details.
+
+   ----------------------------------------------------------------
+
+   This file is part of Valgrind, a dynamic binary instrumentation
+   framework.
+
+   Copyright (C) 2000-2008 Julian Seward.  All rights reserved.
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions
+   are met:
+
+   1. Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+   2. The origin of this software must not be misrepresented; you must 
+      not claim that you wrote the original software.  If you use this 
+      software in a product, an acknowledgment in the product 
+      documentation would be appreciated but is not required.
+
+   3. Altered source versions must be plainly marked as such, and must
+      not be misrepresented as being the original software.
+
+   4. The name of the author may not be used to endorse or promote 
+      products derived from this software without specific prior written 
+      permission.
+
+   THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
+   OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+   WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+   ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+   DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+   DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+   GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+   WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+   ----------------------------------------------------------------
+
+   Notice that the above BSD-style license applies to this one file
+   (valgrind.h) only.  The entire rest of Valgrind is licensed under
+   the terms of the GNU General Public License, version 2.  See the
+   COPYING file in the source distribution for details.
+
+   ---------------------------------------------------------------- 
+*/
+
+
+/* This file is for inclusion into client (your!) code.
+
+   You can use these macros to manipulate and query Valgrind's 
+   execution inside your own programs.
+
+   The resulting executables will still run without Valgrind, just a
+   little bit more slowly than they otherwise would, but otherwise
+   unchanged.  When not running on valgrind, each client request
+   consumes very few (eg. 7) instructions, so the resulting performance
+   loss is negligible unless you plan to execute client requests
+   millions of times per second.  Nevertheless, if that is still a
+   problem, you can compile with the NVALGRIND symbol defined (gcc
+   -DNVALGRIND) so that client requests are not even compiled in.  */
+
+#ifndef __VALGRIND_H
+#define __VALGRIND_H
+
+#include <stdarg.h>
+
+/* Nb: this file might be included in a file compiled with -ansi.  So
+   we can't use C++ style "//" comments nor the "asm" keyword (instead
+   use "__asm__"). */
+
+/* Derive some tags indicating what the target platform is.  Note
+   that in this file we're using the compiler's CPP symbols for
+   identifying architectures, which are different to the ones we use
+   within the rest of Valgrind.  Note, __powerpc__ is active for both
+   32 and 64-bit PPC, whereas __powerpc64__ is only active for the
+   latter (on Linux, that is). */
+#undef PLAT_x86_linux
+#undef PLAT_amd64_linux
+#undef PLAT_ppc32_linux
+#undef PLAT_ppc64_linux
+#undef PLAT_ppc32_aix5
+#undef PLAT_ppc64_aix5
+
+#if !defined(_AIX) && defined(__i386__)
+#  define PLAT_x86_linux 1
+#elif !defined(_AIX) && defined(__x86_64__)
+#  define PLAT_amd64_linux 1
+#elif !defined(_AIX) && defined(__powerpc__) && !defined(__powerpc64__)
+#  define PLAT_ppc32_linux 1
+#elif !defined(_AIX) && defined(__powerpc__) && defined(__powerpc64__)
+#  define PLAT_ppc64_linux 1
+#elif defined(_AIX) && defined(__64BIT__)
+#  define PLAT_ppc64_aix5 1
+#elif defined(_AIX) && !defined(__64BIT__)
+#  define PLAT_ppc32_aix5 1
+#endif
+
+
+/* If we're not compiling for our target platform, don't generate
+   any inline asms.  */
+#if !defined(PLAT_x86_linux) && !defined(PLAT_amd64_linux) \
+    && !defined(PLAT_ppc32_linux) && !defined(PLAT_ppc64_linux) \
+    && !defined(PLAT_ppc32_aix5) && !defined(PLAT_ppc64_aix5)
+#  if !defined(NVALGRIND)
+#    define NVALGRIND 1
+#  endif
+#endif
+
+
+/* ------------------------------------------------------------------ */
+/* ARCHITECTURE SPECIFICS for SPECIAL INSTRUCTIONS.  There is nothing */
+/* in here of use to end-users -- skip to the next section.           */
+/* ------------------------------------------------------------------ */
+
+#if defined(NVALGRIND)
+
+/* Define NVALGRIND to completely remove the Valgrind magic sequence
+   from the compiled code (analogous to NDEBUG's effects on
+   assert()) */
+#define VALGRIND_DO_CLIENT_REQUEST(                               \
+        _zzq_rlval, _zzq_default, _zzq_request,                   \
+        _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
+   {                                                              \
+      (_zzq_rlval) = (_zzq_default);                              \
+   }
+
+#else  /* ! NVALGRIND */
+
+/* The following defines the magic code sequences which the JITter
+   spots and handles magically.  Don't look too closely at them as
+   they will rot your brain.
+
+   The assembly code sequences for all architectures is in this one
+   file.  This is because this file must be stand-alone, and we don't
+   want to have multiple files.
+
+   For VALGRIND_DO_CLIENT_REQUEST, we must ensure that the default
+   value gets put in the return slot, so that everything works when
+   this is executed not under Valgrind.  Args are passed in a memory
+   block, and so there's no intrinsic limit to the number that could
+   be passed, but it's currently five.
+   
+   The macro args are: 
+      _zzq_rlval    result lvalue
+      _zzq_default  default value (result returned when running on real CPU)
+      _zzq_request  request code
+      _zzq_arg1..5  request params
+
+   The other two macros are used to support function wrapping, and are
+   a lot simpler.  VALGRIND_GET_NR_CONTEXT returns the value of the
+   guest's NRADDR pseudo-register and whatever other information is
+   needed to safely run the call original from the wrapper: on
+   ppc64-linux, the R2 value at the divert point is also needed.  This
+   information is abstracted into a user-visible type, OrigFn.
+
+   VALGRIND_CALL_NOREDIR_* behaves the same as the following on the
+   guest, but guarantees that the branch instruction will not be
+   redirected: x86: call *%eax, amd64: call *%rax, ppc32/ppc64:
+   branch-and-link-to-r11.  VALGRIND_CALL_NOREDIR is just text, not a
+   complete inline asm, since it needs to be combined with more magic
+   inline asm stuff to be useful.
+*/
+
+/* ------------------------- x86-linux ------------------------- */
+
+#if defined(PLAT_x86_linux)
+
+typedef
+   struct { 
+      unsigned int nraddr; /* where's the code? */
+   }
+   OrigFn;
+
+#define __SPECIAL_INSTRUCTION_PREAMBLE                            \
+                     "roll $3,  %%edi ; roll $13, %%edi\n\t"      \
+                     "roll $29, %%edi ; roll $19, %%edi\n\t"
+
+#define VALGRIND_DO_CLIENT_REQUEST(                               \
+        _zzq_rlval, _zzq_default, _zzq_request,                   \
+        _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
+  { volatile unsigned int _zzq_args[6];                           \
+    volatile unsigned int _zzq_result;                            \
+    _zzq_args[0] = (unsigned int)(_zzq_request);                  \
+    _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
+    _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
+    _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
+    _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
+    _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
+    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
+                     /* %EDX = client_request ( %EAX ) */         \
+                     "xchgl %%ebx,%%ebx"                          \
+                     : "=d" (_zzq_result)                         \
+                     : "a" (&_zzq_args[0]), "0" (_zzq_default)    \
+                     : "cc", "memory"                             \
+                    );                                            \
+    _zzq_rlval = _zzq_result;                                     \
+  }
+
+#define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
+  { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
+    volatile unsigned int __addr;                                 \
+    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
+                     /* %EAX = guest_NRADDR */                    \
+                     "xchgl %%ecx,%%ecx"                          \
+                     : "=a" (__addr)                              \
+                     :                                            \
+                     : "cc", "memory"                             \
+                    );                                            \
+    _zzq_orig->nraddr = __addr;                                   \
+  }
+
+#define VALGRIND_CALL_NOREDIR_EAX                                 \
+                     __SPECIAL_INSTRUCTION_PREAMBLE               \
+                     /* call-noredir *%EAX */                     \
+                     "xchgl %%edx,%%edx\n\t"
+#endif /* PLAT_x86_linux */
+
+/* ------------------------ amd64-linux ------------------------ */
+
+#if defined(PLAT_amd64_linux)
+
+typedef
+   struct { 
+      unsigned long long int nraddr; /* where's the code? */
+   }
+   OrigFn;
+
+#define __SPECIAL_INSTRUCTION_PREAMBLE                            \
+                     "rolq $3,  %%rdi ; rolq $13, %%rdi\n\t"      \
+                     "rolq $61, %%rdi ; rolq $51, %%rdi\n\t"
+
+#define VALGRIND_DO_CLIENT_REQUEST(                               \
+        _zzq_rlval, _zzq_default, _zzq_request,                   \
+        _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
+  { volatile unsigned long long int _zzq_args[6];                 \
+    volatile unsigned long long int _zzq_result;                  \
+    _zzq_args[0] = (unsigned long long int)(_zzq_request);        \
+    _zzq_args[1] = (unsigned long long int)(_zzq_arg1);           \
+    _zzq_args[2] = (unsigned long long int)(_zzq_arg2);           \
+    _zzq_args[3] = (unsigned long long int)(_zzq_arg3);           \
+    _zzq_args[4] = (unsigned long long int)(_zzq_arg4);           \
+    _zzq_args[5] = (unsigned long long int)(_zzq_arg5);           \
+    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
+                     /* %RDX = client_request ( %RAX ) */         \
+                     "xchgq %%rbx,%%rbx"                          \
+                     : "=d" (_zzq_result)                         \
+                     : "a" (&_zzq_args[0]), "0" (_zzq_default)    \
+                     : "cc", "memory"                             \
+                    );                                            \
+    _zzq_rlval = _zzq_result;                                     \
+  }
+
+#define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
+  { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
+    volatile unsigned long long int __addr;                       \
+    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
+                     /* %RAX = guest_NRADDR */                    \
+                     "xchgq %%rcx,%%rcx"                          \
+                     : "=a" (__addr)                              \
+                     :                                            \
+                     : "cc", "memory"                             \
+                    );                                            \
+    _zzq_orig->nraddr = __addr;                                   \
+  }
+
+#define VALGRIND_CALL_NOREDIR_RAX                                 \
+                     __SPECIAL_INSTRUCTION_PREAMBLE               \
+                     /* call-noredir *%RAX */                     \
+                     "xchgq %%rdx,%%rdx\n\t"
+#endif /* PLAT_amd64_linux */
+
+/* ------------------------ ppc32-linux ------------------------ */
+
+#if defined(PLAT_ppc32_linux)
+
+typedef
+   struct { 
+      unsigned int nraddr; /* where's the code? */
+   }
+   OrigFn;
+
+#define __SPECIAL_INSTRUCTION_PREAMBLE                            \
+                     "rlwinm 0,0,3,0,0  ; rlwinm 0,0,13,0,0\n\t"  \
+                     "rlwinm 0,0,29,0,0 ; rlwinm 0,0,19,0,0\n\t"
+
+#define VALGRIND_DO_CLIENT_REQUEST(                               \
+        _zzq_rlval, _zzq_default, _zzq_request,                   \
+        _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
+                                                                  \
+  {          unsigned int  _zzq_args[6];                          \
+             unsigned int  _zzq_result;                           \
+             unsigned int* _zzq_ptr;                              \
+    _zzq_args[0] = (unsigned int)(_zzq_request);                  \
+    _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
+    _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
+    _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
+    _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
+    _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
+    _zzq_ptr = _zzq_args;                                         \
+    __asm__ volatile("mr 3,%1\n\t" /*default*/                    \
+                     "mr 4,%2\n\t" /*ptr*/                        \
+                     __SPECIAL_INSTRUCTION_PREAMBLE               \
+                     /* %R3 = client_request ( %R4 ) */           \
+                     "or 1,1,1\n\t"                               \
+                     "mr %0,3"     /*result*/                     \
+                     : "=b" (_zzq_result)                         \
+                     : "b" (_zzq_default), "b" (_zzq_ptr)         \
+                     : "cc", "memory", "r3", "r4");               \
+    _zzq_rlval = _zzq_result;                                     \
+  }
+
+#define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
+  { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
+    unsigned int __addr;                                          \
+    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
+                     /* %R3 = guest_NRADDR */                     \
+                     "or 2,2,2\n\t"                               \
+                     "mr %0,3"                                    \
+                     : "=b" (__addr)                              \
+                     :                                            \
+                     : "cc", "memory", "r3"                       \
+                    );                                            \
+    _zzq_orig->nraddr = __addr;                                   \
+  }
+
+#define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
+                     __SPECIAL_INSTRUCTION_PREAMBLE               \
+                     /* branch-and-link-to-noredir *%R11 */       \
+                     "or 3,3,3\n\t"
+#endif /* PLAT_ppc32_linux */
+
+/* ------------------------ ppc64-linux ------------------------ */
+
+#if defined(PLAT_ppc64_linux)
+
+typedef
+   struct { 
+      unsigned long long int nraddr; /* where's the code? */
+      unsigned long long int r2;  /* what tocptr do we need? */
+   }
+   OrigFn;
+
+#define __SPECIAL_INSTRUCTION_PREAMBLE                            \
+                     "rotldi 0,0,3  ; rotldi 0,0,13\n\t"          \
+                     "rotldi 0,0,61 ; rotldi 0,0,51\n\t"
+
+#define VALGRIND_DO_CLIENT_REQUEST(                               \
+        _zzq_rlval, _zzq_default, _zzq_request,                   \
+        _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
+                                                                  \
+  {          unsigned long long int  _zzq_args[6];                \
+    register unsigned long long int  _zzq_result __asm__("r3");   \
+    register unsigned long long int* _zzq_ptr __asm__("r4");      \
+    _zzq_args[0] = (unsigned long long int)(_zzq_request);        \
+    _zzq_args[1] = (unsigned long long int)(_zzq_arg1);           \
+    _zzq_args[2] = (unsigned long long int)(_zzq_arg2);           \
+    _zzq_args[3] = (unsigned long long int)(_zzq_arg3);           \
+    _zzq_args[4] = (unsigned long long int)(_zzq_arg4);           \
+    _zzq_args[5] = (unsigned long long int)(_zzq_arg5);           \
+    _zzq_ptr = _zzq_args;                                         \
+    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
+                     /* %R3 = client_request ( %R4 ) */           \
+                     "or 1,1,1"                                   \
+                     : "=r" (_zzq_result)                         \
+                     : "0" (_zzq_default), "r" (_zzq_ptr)         \
+                     : "cc", "memory");                           \
+    _zzq_rlval = _zzq_result;                                     \
+  }
+
+#define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
+  { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
+    register unsigned long long int __addr __asm__("r3");         \
+    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
+                     /* %R3 = guest_NRADDR */                     \
+                     "or 2,2,2"                                   \
+                     : "=r" (__addr)                              \
+                     :                                            \
+                     : "cc", "memory"                             \
+                    );                                            \
+    _zzq_orig->nraddr = __addr;                                   \
+    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
+                     /* %R3 = guest_NRADDR_GPR2 */                \
+                     "or 4,4,4"                                   \
+                     : "=r" (__addr)                              \
+                     :                                            \
+                     : "cc", "memory"                             \
+                    );                                            \
+    _zzq_orig->r2 = __addr;                                       \
+  }
+
+#define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
+                     __SPECIAL_INSTRUCTION_PREAMBLE               \
+                     /* branch-and-link-to-noredir *%R11 */       \
+                     "or 3,3,3\n\t"
+
+#endif /* PLAT_ppc64_linux */
+
+/* ------------------------ ppc32-aix5 ------------------------- */
+
+#if defined(PLAT_ppc32_aix5)
+
+typedef
+   struct { 
+      unsigned int nraddr; /* where's the code? */
+      unsigned int r2;  /* what tocptr do we need? */
+   }
+   OrigFn;
+
+#define __SPECIAL_INSTRUCTION_PREAMBLE                            \
+                     "rlwinm 0,0,3,0,0  ; rlwinm 0,0,13,0,0\n\t"  \
+                     "rlwinm 0,0,29,0,0 ; rlwinm 0,0,19,0,0\n\t"
+
+#define VALGRIND_DO_CLIENT_REQUEST(                               \
+        _zzq_rlval, _zzq_default, _zzq_request,                   \
+        _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
+                                                                  \
+  {          unsigned int  _zzq_args[7];                          \
+    register unsigned int  _zzq_result;                           \
+    register unsigned int* _zzq_ptr;                              \
+    _zzq_args[0] = (unsigned int)(_zzq_request);                  \
+    _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
+    _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
+    _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
+    _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
+    _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
+    _zzq_args[6] = (unsigned int)(_zzq_default);                  \
+    _zzq_ptr = _zzq_args;                                         \
+    __asm__ volatile("mr 4,%1\n\t"                                \
+                     "lwz 3, 24(4)\n\t"                           \
+                     __SPECIAL_INSTRUCTION_PREAMBLE               \
+                     /* %R3 = client_request ( %R4 ) */           \
+                     "or 1,1,1\n\t"                               \
+                     "mr %0,3"                                    \
+                     : "=b" (_zzq_result)                         \
+                     : "b" (_zzq_ptr)                             \
+                     : "r3", "r4", "cc", "memory");               \
+    _zzq_rlval = _zzq_result;                                     \
+  }
+
+#define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
+  { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
+    register unsigned int __addr;                                 \
+    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
+                     /* %R3 = guest_NRADDR */                     \
+                     "or 2,2,2\n\t"                               \
+                     "mr %0,3"                                    \
+                     : "=b" (__addr)                              \
+                     :                                            \
+                     : "r3", "cc", "memory"                       \
+                    );                                            \
+    _zzq_orig->nraddr = __addr;                                   \
+    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
+                     /* %R3 = guest_NRADDR_GPR2 */                \
+                     "or 4,4,4\n\t"                               \
+                     "mr %0,3"                                    \
+                     : "=b" (__addr)                              \
+                     :                                            \
+                     : "r3", "cc", "memory"                       \
+                    );                                            \
+    _zzq_orig->r2 = __addr;                                       \
+  }
+
+#define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
+                     __SPECIAL_INSTRUCTION_PREAMBLE               \
+                     /* branch-and-link-to-noredir *%R11 */       \
+                     "or 3,3,3\n\t"
+
+#endif /* PLAT_ppc32_aix5 */
+
+/* ------------------------ ppc64-aix5 ------------------------- */
+
+#if defined(PLAT_ppc64_aix5)
+
+typedef
+   struct { 
+      unsigned long long int nraddr; /* where's the code? */
+      unsigned long long int r2;  /* what tocptr do we need? */
+   }
+   OrigFn;
+
+#define __SPECIAL_INSTRUCTION_PREAMBLE                            \
+                     "rotldi 0,0,3  ; rotldi 0,0,13\n\t"          \
+                     "rotldi 0,0,61 ; rotldi 0,0,51\n\t"
+
+#define VALGRIND_DO_CLIENT_REQUEST(                               \
+        _zzq_rlval, _zzq_default, _zzq_request,                   \
+        _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
+                                                                  \
+  {          unsigned long long int  _zzq_args[7];                \
+    register unsigned long long int  _zzq_result;                 \
+    register unsigned long long int* _zzq_ptr;                    \
+    _zzq_args[0] = (unsigned int long long)(_zzq_request);        \
+    _zzq_args[1] = (unsigned int long long)(_zzq_arg1);           \
+    _zzq_args[2] = (unsigned int long long)(_zzq_arg2);           \
+    _zzq_args[3] = (unsigned int long long)(_zzq_arg3);           \
+    _zzq_args[4] = (unsigned int long long)(_zzq_arg4);           \
+    _zzq_args[5] = (unsigned int long long)(_zzq_arg5);           \
+    _zzq_args[6] = (unsigned int long long)(_zzq_default);        \
+    _zzq_ptr = _zzq_args;                                         \
+    __asm__ volatile("mr 4,%1\n\t"                                \
+                     "ld 3, 48(4)\n\t"                            \
+                     __SPECIAL_INSTRUCTION_PREAMBLE               \
+                     /* %R3 = client_request ( %R4 ) */           \
+                     "or 1,1,1\n\t"                               \
+                     "mr %0,3"                                    \
+                     : "=b" (_zzq_result)                         \
+                     : "b" (_zzq_ptr)                             \
+                     : "r3", "r4", "cc", "memory");               \
+    _zzq_rlval = _zzq_result;                                     \
+  }
+
+#define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
+  { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
+    register unsigned long long int __addr;                       \
+    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
+                     /* %R3 = guest_NRADDR */                     \
+                     "or 2,2,2\n\t"                               \
+                     "mr %0,3"                                    \
+                     : "=b" (__addr)                              \
+                     :                                            \
+                     : "r3", "cc", "memory"                       \
+                    );                                            \
+    _zzq_orig->nraddr = __addr;                                   \
+    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
+                     /* %R3 = guest_NRADDR_GPR2 */                \
+                     "or 4,4,4\n\t"                               \
+                     "mr %0,3"                                    \
+                     : "=b" (__addr)                              \
+                     :                                            \
+                     : "r3", "cc", "memory"                       \
+                    );                                            \
+    _zzq_orig->r2 = __addr;                                       \
+  }
+
+#define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
+                     __SPECIAL_INSTRUCTION_PREAMBLE               \
+                     /* branch-and-link-to-noredir *%R11 */       \
+                     "or 3,3,3\n\t"
+
+#endif /* PLAT_ppc64_aix5 */
+
+/* Insert assembly code for other platforms here... */
+
+#endif /* NVALGRIND */
+
+
+/* ------------------------------------------------------------------ */
+/* PLATFORM SPECIFICS for FUNCTION WRAPPING.  This is all very        */
+/* ugly.  It's the least-worst tradeoff I can think of.               */
+/* ------------------------------------------------------------------ */
+
+/* This section defines magic (a.k.a appalling-hack) macros for doing
+   guaranteed-no-redirection macros, so as to get from function
+   wrappers to the functions they are wrapping.  The whole point is to
+   construct standard call sequences, but to do the call itself with a
+   special no-redirect call pseudo-instruction that the JIT
+   understands and handles specially.  This section is long and
+   repetitious, and I can't see a way to make it shorter.
+
+   The naming scheme is as follows:
+
+      CALL_FN_{W,v}_{v,W,WW,WWW,WWWW,5W,6W,7W,etc}
+
+   'W' stands for "word" and 'v' for "void".  Hence there are
+   different macros for calling arity 0, 1, 2, 3, 4, etc, functions,
+   and for each, the possibility of returning a word-typed result, or
+   no result.
+*/
+
+/* Use these to write the name of your wrapper.  NOTE: duplicates
+   VG_WRAP_FUNCTION_Z{U,Z} in pub_tool_redir.h. */
+
+#define I_WRAP_SONAME_FNNAME_ZU(soname,fnname)                    \
+   _vgwZU_##soname##_##fnname
+
+#define I_WRAP_SONAME_FNNAME_ZZ(soname,fnname)                    \
+   _vgwZZ_##soname##_##fnname
+
+/* Use this macro from within a wrapper function to collect the
+   context (address and possibly other info) of the original function.
+   Once you have that you can then use it in one of the CALL_FN_
+   macros.  The type of the argument _lval is OrigFn. */
+#define VALGRIND_GET_ORIG_FN(_lval)  VALGRIND_GET_NR_CONTEXT(_lval)
+
+/* Derivatives of the main macros below, for calling functions
+   returning void. */
+
+#define CALL_FN_v_v(fnptr)                                        \
+   do { volatile unsigned long _junk;                             \
+        CALL_FN_W_v(_junk,fnptr); } while (0)
+
+#define CALL_FN_v_W(fnptr, arg1)                                  \
+   do { volatile unsigned long _junk;                             \
+        CALL_FN_W_W(_junk,fnptr,arg1); } while (0)
+
+#define CALL_FN_v_WW(fnptr, arg1,arg2)                            \
+   do { volatile unsigned long _junk;                             \
+        CALL_FN_W_WW(_junk,fnptr,arg1,arg2); } while (0)
+
+#define CALL_FN_v_WWW(fnptr, arg1,arg2,arg3)                      \
+   do { volatile unsigned long _junk;                             \
+        CALL_FN_W_WWW(_junk,fnptr,arg1,arg2,arg3); } while (0)
+
+/* ------------------------- x86-linux ------------------------- */
+
+#if defined(PLAT_x86_linux)
+
+/* These regs are trashed by the hidden call.  No need to mention eax
+   as gcc can already see that, plus causes gcc to bomb. */
+#define __CALLER_SAVED_REGS /*"eax"*/ "ecx", "edx"
+
+/* These CALL_FN_ macros assume that on x86-linux, sizeof(unsigned
+   long) == 4. */
+
+#define CALL_FN_W_v(lval, orig)                                   \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[1];                          \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      __asm__ volatile(                                           \
+         "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
+         VALGRIND_CALL_NOREDIR_EAX                                \
+         : /*out*/   "=a" (_res)                                  \
+         : /*in*/    "a" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_W(lval, orig, arg1)                             \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[2];                          \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)(arg1);                         \
+      __asm__ volatile(                                           \
+         "pushl 4(%%eax)\n\t"                                     \
+         "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
+         VALGRIND_CALL_NOREDIR_EAX                                \
+         "addl $4, %%esp\n"                                       \
+         : /*out*/   "=a" (_res)                                  \
+         : /*in*/    "a" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3];                          \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)(arg1);                         \
+      _argvec[2] = (unsigned long)(arg2);                         \
+      __asm__ volatile(                                           \
+         "pushl 8(%%eax)\n\t"                                     \
+         "pushl 4(%%eax)\n\t"                                     \
+         "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
+         VALGRIND_CALL_NOREDIR_EAX                                \
+         "addl $8, %%esp\n"                                       \
+         : /*out*/   "=a" (_res)                                  \
+         : /*in*/    "a" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[4];                          \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)(arg1);                         \
+      _argvec[2] = (unsigned long)(arg2);                         \
+      _argvec[3] = (unsigned long)(arg3);                         \
+      __asm__ volatile(                                           \
+         "pushl 12(%%eax)\n\t"                                    \
+         "pushl 8(%%eax)\n\t"                                     \
+         "pushl 4(%%eax)\n\t"                                     \
+         "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
+         VALGRIND_CALL_NOREDIR_EAX                                \
+         "addl $12, %%esp\n"                                      \
+         : /*out*/   "=a" (_res)                                  \
+         : /*in*/    "a" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[5];                          \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)(arg1);                         \
+      _argvec[2] = (unsigned long)(arg2);                         \
+      _argvec[3] = (unsigned long)(arg3);                         \
+      _argvec[4] = (unsigned long)(arg4);                         \
+      __asm__ volatile(                                           \
+         "pushl 16(%%eax)\n\t"                                    \
+         "pushl 12(%%eax)\n\t"                                    \
+         "pushl 8(%%eax)\n\t"                                     \
+         "pushl 4(%%eax)\n\t"                                     \
+         "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
+         VALGRIND_CALL_NOREDIR_EAX                                \
+         "addl $16, %%esp\n"                                      \
+         : /*out*/   "=a" (_res)                                  \
+         : /*in*/    "a" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[6];                          \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)(arg1);                         \
+      _argvec[2] = (unsigned long)(arg2);                         \
+      _argvec[3] = (unsigned long)(arg3);                         \
+      _argvec[4] = (unsigned long)(arg4);                         \
+      _argvec[5] = (unsigned long)(arg5);                         \
+      __asm__ volatile(                                           \
+         "pushl 20(%%eax)\n\t"                                    \
+         "pushl 16(%%eax)\n\t"                                    \
+         "pushl 12(%%eax)\n\t"                                    \
+         "pushl 8(%%eax)\n\t"                                     \
+         "pushl 4(%%eax)\n\t"                                     \
+         "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
+         VALGRIND_CALL_NOREDIR_EAX                                \
+         "addl $20, %%esp\n"                                      \
+         : /*out*/   "=a" (_res)                                  \
+         : /*in*/    "a" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[7];                          \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)(arg1);                         \
+      _argvec[2] = (unsigned long)(arg2);                         \
+      _argvec[3] = (unsigned long)(arg3);                         \
+      _argvec[4] = (unsigned long)(arg4);                         \
+      _argvec[5] = (unsigned long)(arg5);                         \
+      _argvec[6] = (unsigned long)(arg6);                         \
+      __asm__ volatile(                                           \
+         "pushl 24(%%eax)\n\t"                                    \
+         "pushl 20(%%eax)\n\t"                                    \
+         "pushl 16(%%eax)\n\t"                                    \
+         "pushl 12(%%eax)\n\t"                                    \
+         "pushl 8(%%eax)\n\t"                                     \
+         "pushl 4(%%eax)\n\t"                                     \
+         "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
+         VALGRIND_CALL_NOREDIR_EAX                                \
+         "addl $24, %%esp\n"                                      \
+         : /*out*/   "=a" (_res)                                  \
+         : /*in*/    "a" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
+                                 arg7)                            \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[8];                          \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)(arg1);                         \
+      _argvec[2] = (unsigned long)(arg2);                         \
+      _argvec[3] = (unsigned long)(arg3);                         \
+      _argvec[4] = (unsigned long)(arg4);                         \
+      _argvec[5] = (unsigned long)(arg5);                         \
+      _argvec[6] = (unsigned long)(arg6);                         \
+      _argvec[7] = (unsigned long)(arg7);                         \
+      __asm__ volatile(                                           \
+         "pushl 28(%%eax)\n\t"                                    \
+         "pushl 24(%%eax)\n\t"                                    \
+         "pushl 20(%%eax)\n\t"                                    \
+         "pushl 16(%%eax)\n\t"                                    \
+         "pushl 12(%%eax)\n\t"                                    \
+         "pushl 8(%%eax)\n\t"                                     \
+         "pushl 4(%%eax)\n\t"                                     \
+         "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
+         VALGRIND_CALL_NOREDIR_EAX                                \
+         "addl $28, %%esp\n"                                      \
+         : /*out*/   "=a" (_res)                                  \
+         : /*in*/    "a" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
+                                 arg7,arg8)                       \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[9];                          \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)(arg1);                         \
+      _argvec[2] = (unsigned long)(arg2);                         \
+      _argvec[3] = (unsigned long)(arg3);                         \
+      _argvec[4] = (unsigned long)(arg4);                         \
+      _argvec[5] = (unsigned long)(arg5);                         \
+      _argvec[6] = (unsigned long)(arg6);                         \
+      _argvec[7] = (unsigned long)(arg7);                         \
+      _argvec[8] = (unsigned long)(arg8);                         \
+      __asm__ volatile(                                           \
+         "pushl 32(%%eax)\n\t"                                    \
+         "pushl 28(%%eax)\n\t"                                    \
+         "pushl 24(%%eax)\n\t"                                    \
+         "pushl 20(%%eax)\n\t"                                    \
+         "pushl 16(%%eax)\n\t"                                    \
+         "pushl 12(%%eax)\n\t"                                    \
+         "pushl 8(%%eax)\n\t"                                     \
+         "pushl 4(%%eax)\n\t"                                     \
+         "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
+         VALGRIND_CALL_NOREDIR_EAX                                \
+         "addl $32, %%esp\n"                                      \
+         : /*out*/   "=a" (_res)                                  \
+         : /*in*/    "a" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
+                                 arg7,arg8,arg9)                  \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[10];                         \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)(arg1);                         \
+      _argvec[2] = (unsigned long)(arg2);                         \
+      _argvec[3] = (unsigned long)(arg3);                         \
+      _argvec[4] = (unsigned long)(arg4);                         \
+      _argvec[5] = (unsigned long)(arg5);                         \
+      _argvec[6] = (unsigned long)(arg6);                         \
+      _argvec[7] = (unsigned long)(arg7);                         \
+      _argvec[8] = (unsigned long)(arg8);                         \
+      _argvec[9] = (unsigned long)(arg9);                         \
+      __asm__ volatile(                                           \
+         "pushl 36(%%eax)\n\t"                                    \
+         "pushl 32(%%eax)\n\t"                                    \
+         "pushl 28(%%eax)\n\t"                                    \
+         "pushl 24(%%eax)\n\t"                                    \
+         "pushl 20(%%eax)\n\t"                                    \
+         "pushl 16(%%eax)\n\t"                                    \
+         "pushl 12(%%eax)\n\t"                                    \
+         "pushl 8(%%eax)\n\t"                                     \
+         "pushl 4(%%eax)\n\t"                                     \
+         "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
+         VALGRIND_CALL_NOREDIR_EAX                                \
+         "addl $36, %%esp\n"                                      \
+         : /*out*/   "=a" (_res)                                  \
+         : /*in*/    "a" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
+                                  arg7,arg8,arg9,arg10)           \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[11];                         \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)(arg1);                         \
+      _argvec[2] = (unsigned long)(arg2);                         \
+      _argvec[3] = (unsigned long)(arg3);                         \
+      _argvec[4] = (unsigned long)(arg4);                         \
+      _argvec[5] = (unsigned long)(arg5);                         \
+      _argvec[6] = (unsigned long)(arg6);                         \
+      _argvec[7] = (unsigned long)(arg7);                         \
+      _argvec[8] = (unsigned long)(arg8);                         \
+      _argvec[9] = (unsigned long)(arg9);                         \
+      _argvec[10] = (unsigned long)(arg10);                       \
+      __asm__ volatile(                                           \
+         "pushl 40(%%eax)\n\t"                                    \
+         "pushl 36(%%eax)\n\t"                                    \
+         "pushl 32(%%eax)\n\t"                                    \
+         "pushl 28(%%eax)\n\t"                                    \
+         "pushl 24(%%eax)\n\t"                                    \
+         "pushl 20(%%eax)\n\t"                                    \
+         "pushl 16(%%eax)\n\t"                                    \
+         "pushl 12(%%eax)\n\t"                                    \
+         "pushl 8(%%eax)\n\t"                                     \
+         "pushl 4(%%eax)\n\t"                                     \
+         "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
+         VALGRIND_CALL_NOREDIR_EAX                                \
+         "addl $40, %%esp\n"                                      \
+         : /*out*/   "=a" (_res)                                  \
+         : /*in*/    "a" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
+                                  arg6,arg7,arg8,arg9,arg10,      \
+                                  arg11)                          \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[12];                         \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)(arg1);                         \
+      _argvec[2] = (unsigned long)(arg2);                         \
+      _argvec[3] = (unsigned long)(arg3);                         \
+      _argvec[4] = (unsigned long)(arg4);                         \
+      _argvec[5] = (unsigned long)(arg5);                         \
+      _argvec[6] = (unsigned long)(arg6);                         \
+      _argvec[7] = (unsigned long)(arg7);                         \
+      _argvec[8] = (unsigned long)(arg8);                         \
+      _argvec[9] = (unsigned long)(arg9);                         \
+      _argvec[10] = (unsigned long)(arg10);                       \
+      _argvec[11] = (unsigned long)(arg11);                       \
+      __asm__ volatile(                                           \
+         "pushl 44(%%eax)\n\t"                                    \
+         "pushl 40(%%eax)\n\t"                                    \
+         "pushl 36(%%eax)\n\t"                                    \
+         "pushl 32(%%eax)\n\t"                                    \
+         "pushl 28(%%eax)\n\t"                                    \
+         "pushl 24(%%eax)\n\t"                                    \
+         "pushl 20(%%eax)\n\t"                                    \
+         "pushl 16(%%eax)\n\t"                                    \
+         "pushl 12(%%eax)\n\t"                                    \
+         "pushl 8(%%eax)\n\t"                                     \
+         "pushl 4(%%eax)\n\t"                                     \
+         "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
+         VALGRIND_CALL_NOREDIR_EAX                                \
+         "addl $44, %%esp\n"                                      \
+         : /*out*/   "=a" (_res)                                  \
+         : /*in*/    "a" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
+                                  arg6,arg7,arg8,arg9,arg10,      \
+                                  arg11,arg12)                    \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[13];                         \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)(arg1);                         \
+      _argvec[2] = (unsigned long)(arg2);                         \
+      _argvec[3] = (unsigned long)(arg3);                         \
+      _argvec[4] = (unsigned long)(arg4);                         \
+      _argvec[5] = (unsigned long)(arg5);                         \
+      _argvec[6] = (unsigned long)(arg6);                         \
+      _argvec[7] = (unsigned long)(arg7);                         \
+      _argvec[8] = (unsigned long)(arg8);                         \
+      _argvec[9] = (unsigned long)(arg9);                         \
+      _argvec[10] = (unsigned long)(arg10);                       \
+      _argvec[11] = (unsigned long)(arg11);                       \
+      _argvec[12] = (unsigned long)(arg12);                       \
+      __asm__ volatile(                                           \
+         "pushl 48(%%eax)\n\t"                                    \
+         "pushl 44(%%eax)\n\t"                                    \
+         "pushl 40(%%eax)\n\t"                                    \
+         "pushl 36(%%eax)\n\t"                                    \
+         "pushl 32(%%eax)\n\t"                                    \
+         "pushl 28(%%eax)\n\t"                                    \
+         "pushl 24(%%eax)\n\t"                                    \
+         "pushl 20(%%eax)\n\t"                                    \
+         "pushl 16(%%eax)\n\t"                                    \
+         "pushl 12(%%eax)\n\t"                                    \
+         "pushl 8(%%eax)\n\t"                                     \
+         "pushl 4(%%eax)\n\t"                                     \
+         "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
+         VALGRIND_CALL_NOREDIR_EAX                                \
+         "addl $48, %%esp\n"                                      \
+         : /*out*/   "=a" (_res)                                  \
+         : /*in*/    "a" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#endif /* PLAT_x86_linux */
+
+/* ------------------------ amd64-linux ------------------------ */
+
+#if defined(PLAT_amd64_linux)
+
+/* ARGREGS: rdi rsi rdx rcx r8 r9 (the rest on stack in R-to-L order) */
+
+/* These regs are trashed by the hidden call. */
+#define __CALLER_SAVED_REGS /*"rax",*/ "rcx", "rdx", "rsi",       \
+                            "rdi", "r8", "r9", "r10", "r11"
+
+/* These CALL_FN_ macros assume that on amd64-linux, sizeof(unsigned
+   long) == 8. */
+
+/* NB 9 Sept 07.  There is a nasty kludge here in all these CALL_FN_
+   macros.  In order not to trash the stack redzone, we need to drop
+   %rsp by 128 before the hidden call, and restore afterwards.  The
+   nastyness is that it is only by luck that the stack still appears
+   to be unwindable during the hidden call - since then the behaviour
+   of any routine using this macro does not match what the CFI data
+   says.  Sigh.
+
+   Why is this important?  Imagine that a wrapper has a stack
+   allocated local, and passes to the hidden call, a pointer to it.
+   Because gcc does not know about the hidden call, it may allocate
+   that local in the redzone.  Unfortunately the hidden call may then
+   trash it before it comes to use it.  So we must step clear of the
+   redzone, for the duration of the hidden call, to make it safe.
+
+   Probably the same problem afflicts the other redzone-style ABIs too
+   (ppc64-linux, ppc32-aix5, ppc64-aix5); but for those, the stack is
+   self describing (none of this CFI nonsense) so at least messing
+   with the stack pointer doesn't give a danger of non-unwindable
+   stack. */
+
+#define CALL_FN_W_v(lval, orig)                                   \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[1];                          \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      __asm__ volatile(                                           \
+         "subq $128,%%rsp\n\t"                                    \
+         "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
+         VALGRIND_CALL_NOREDIR_RAX                                \
+         "addq $128,%%rsp\n\t"                                    \
+         : /*out*/   "=a" (_res)                                  \
+         : /*in*/    "a" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_W(lval, orig, arg1)                             \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[2];                          \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)(arg1);                         \
+      __asm__ volatile(                                           \
+         "subq $128,%%rsp\n\t"                                    \
+         "movq 8(%%rax), %%rdi\n\t"                               \
+         "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
+         VALGRIND_CALL_NOREDIR_RAX                                \
+         "addq $128,%%rsp\n\t"                                    \
+         : /*out*/   "=a" (_res)                                  \
+         : /*in*/    "a" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3];                          \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)(arg1);                         \
+      _argvec[2] = (unsigned long)(arg2);                         \
+      __asm__ volatile(                                           \
+         "subq $128,%%rsp\n\t"                                    \
+         "movq 16(%%rax), %%rsi\n\t"                              \
+         "movq 8(%%rax), %%rdi\n\t"                               \
+         "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
+         VALGRIND_CALL_NOREDIR_RAX                                \
+         "addq $128,%%rsp\n\t"                                    \
+         : /*out*/   "=a" (_res)                                  \
+         : /*in*/    "a" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[4];                          \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)(arg1);                         \
+      _argvec[2] = (unsigned long)(arg2);                         \
+      _argvec[3] = (unsigned long)(arg3);                         \
+      __asm__ volatile(                                           \
+         "subq $128,%%rsp\n\t"                                    \
+         "movq 24(%%rax), %%rdx\n\t"                              \
+         "movq 16(%%rax), %%rsi\n\t"                              \
+         "movq 8(%%rax), %%rdi\n\t"                               \
+         "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
+         VALGRIND_CALL_NOREDIR_RAX                                \
+         "addq $128,%%rsp\n\t"                                    \
+         : /*out*/   "=a" (_res)                                  \
+         : /*in*/    "a" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[5];                          \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)(arg1);                         \
+      _argvec[2] = (unsigned long)(arg2);                         \
+      _argvec[3] = (unsigned long)(arg3);                         \
+      _argvec[4] = (unsigned long)(arg4);                         \
+      __asm__ volatile(                                           \
+         "subq $128,%%rsp\n\t"                                    \
+         "movq 32(%%rax), %%rcx\n\t"                              \
+         "movq 24(%%rax), %%rdx\n\t"                              \
+         "movq 16(%%rax), %%rsi\n\t"                              \
+         "movq 8(%%rax), %%rdi\n\t"                               \
+         "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
+         VALGRIND_CALL_NOREDIR_RAX                                \
+         "addq $128,%%rsp\n\t"                                    \
+         : /*out*/   "=a" (_res)                                  \
+         : /*in*/    "a" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[6];                          \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)(arg1);                         \
+      _argvec[2] = (unsigned long)(arg2);                         \
+      _argvec[3] = (unsigned long)(arg3);                         \
+      _argvec[4] = (unsigned long)(arg4);                         \
+      _argvec[5] = (unsigned long)(arg5);                         \
+      __asm__ volatile(                                           \
+         "subq $128,%%rsp\n\t"                                    \
+         "movq 40(%%rax), %%r8\n\t"                               \
+         "movq 32(%%rax), %%rcx\n\t"                              \
+         "movq 24(%%rax), %%rdx\n\t"                              \
+         "movq 16(%%rax), %%rsi\n\t"                              \
+         "movq 8(%%rax), %%rdi\n\t"                               \
+         "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
+         VALGRIND_CALL_NOREDIR_RAX                                \
+         "addq $128,%%rsp\n\t"                                    \
+         : /*out*/   "=a" (_res)                                  \
+         : /*in*/    "a" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[7];                          \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)(arg1);                         \
+      _argvec[2] = (unsigned long)(arg2);                         \
+      _argvec[3] = (unsigned long)(arg3);                         \
+      _argvec[4] = (unsigned long)(arg4);                         \
+      _argvec[5] = (unsigned long)(arg5);                         \
+      _argvec[6] = (unsigned long)(arg6);                         \
+      __asm__ volatile(                                           \
+         "subq $128,%%rsp\n\t"                                    \
+         "movq 48(%%rax), %%r9\n\t"                               \
+         "movq 40(%%rax), %%r8\n\t"                               \
+         "movq 32(%%rax), %%rcx\n\t"                              \
+         "movq 24(%%rax), %%rdx\n\t"                              \
+         "movq 16(%%rax), %%rsi\n\t"                              \
+         "movq 8(%%rax), %%rdi\n\t"                               \
+         "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
+         "addq $128,%%rsp\n\t"                                    \
+         VALGRIND_CALL_NOREDIR_RAX                                \
+         : /*out*/   "=a" (_res)                                  \
+         : /*in*/    "a" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
+                                 arg7)                            \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[8];                          \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)(arg1);                         \
+      _argvec[2] = (unsigned long)(arg2);                         \
+      _argvec[3] = (unsigned long)(arg3);                         \
+      _argvec[4] = (unsigned long)(arg4);                         \
+      _argvec[5] = (unsigned long)(arg5);                         \
+      _argvec[6] = (unsigned long)(arg6);                         \
+      _argvec[7] = (unsigned long)(arg7);                         \
+      __asm__ volatile(                                           \
+         "subq $128,%%rsp\n\t"                                    \
+         "pushq 56(%%rax)\n\t"                                    \
+         "movq 48(%%rax), %%r9\n\t"                               \
+         "movq 40(%%rax), %%r8\n\t"                               \
+         "movq 32(%%rax), %%rcx\n\t"                              \
+         "movq 24(%%rax), %%rdx\n\t"                              \
+         "movq 16(%%rax), %%rsi\n\t"                              \
+         "movq 8(%%rax), %%rdi\n\t"                               \
+         "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
+         VALGRIND_CALL_NOREDIR_RAX                                \
+         "addq $8, %%rsp\n"                                       \
+         "addq $128,%%rsp\n\t"                                    \
+         : /*out*/   "=a" (_res)                                  \
+         : /*in*/    "a" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
+                                 arg7,arg8)                       \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[9];                          \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)(arg1);                         \
+      _argvec[2] = (unsigned long)(arg2);                         \
+      _argvec[3] = (unsigned long)(arg3);                         \
+      _argvec[4] = (unsigned long)(arg4);                         \
+      _argvec[5] = (unsigned long)(arg5);                         \
+      _argvec[6] = (unsigned long)(arg6);                         \
+      _argvec[7] = (unsigned long)(arg7);                         \
+      _argvec[8] = (unsigned long)(arg8);                         \
+      __asm__ volatile(                                           \
+         "subq $128,%%rsp\n\t"                                    \
+         "pushq 64(%%rax)\n\t"                                    \
+         "pushq 56(%%rax)\n\t"                                    \
+         "movq 48(%%rax), %%r9\n\t"                               \
+         "movq 40(%%rax), %%r8\n\t"                               \
+         "movq 32(%%rax), %%rcx\n\t"                              \
+         "movq 24(%%rax), %%rdx\n\t"                              \
+         "movq 16(%%rax), %%rsi\n\t"                              \
+         "movq 8(%%rax), %%rdi\n\t"                               \
+         "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
+         VALGRIND_CALL_NOREDIR_RAX                                \
+         "addq $16, %%rsp\n"                                      \
+         "addq $128,%%rsp\n\t"                                    \
+         : /*out*/   "=a" (_res)                                  \
+         : /*in*/    "a" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
+                                 arg7,arg8,arg9)                  \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[10];                         \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)(arg1);                         \
+      _argvec[2] = (unsigned long)(arg2);                         \
+      _argvec[3] = (unsigned long)(arg3);                         \
+      _argvec[4] = (unsigned long)(arg4);                         \
+      _argvec[5] = (unsigned long)(arg5);                         \
+      _argvec[6] = (unsigned long)(arg6);                         \
+      _argvec[7] = (unsigned long)(arg7);                         \
+      _argvec[8] = (unsigned long)(arg8);                         \
+      _argvec[9] = (unsigned long)(arg9);                         \
+      __asm__ volatile(                                           \
+         "subq $128,%%rsp\n\t"                                    \
+         "pushq 72(%%rax)\n\t"                                    \
+         "pushq 64(%%rax)\n\t"                                    \
+         "pushq 56(%%rax)\n\t"                                    \
+         "movq 48(%%rax), %%r9\n\t"                               \
+         "movq 40(%%rax), %%r8\n\t"                               \
+         "movq 32(%%rax), %%rcx\n\t"                              \
+         "movq 24(%%rax), %%rdx\n\t"                              \
+         "movq 16(%%rax), %%rsi\n\t"                              \
+         "movq 8(%%rax), %%rdi\n\t"                               \
+         "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
+         VALGRIND_CALL_NOREDIR_RAX                                \
+         "addq $24, %%rsp\n"                                      \
+         "addq $128,%%rsp\n\t"                                    \
+         : /*out*/   "=a" (_res)                                  \
+         : /*in*/    "a" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
+                                  arg7,arg8,arg9,arg10)           \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[11];                         \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)(arg1);                         \
+      _argvec[2] = (unsigned long)(arg2);                         \
+      _argvec[3] = (unsigned long)(arg3);                         \
+      _argvec[4] = (unsigned long)(arg4);                         \
+      _argvec[5] = (unsigned long)(arg5);                         \
+      _argvec[6] = (unsigned long)(arg6);                         \
+      _argvec[7] = (unsigned long)(arg7);                         \
+      _argvec[8] = (unsigned long)(arg8);                         \
+      _argvec[9] = (unsigned long)(arg9);                         \
+      _argvec[10] = (unsigned long)(arg10);                       \
+      __asm__ volatile(                                           \
+         "subq $128,%%rsp\n\t"                                    \
+         "pushq 80(%%rax)\n\t"                                    \
+         "pushq 72(%%rax)\n\t"                                    \
+         "pushq 64(%%rax)\n\t"                                    \
+         "pushq 56(%%rax)\n\t"                                    \
+         "movq 48(%%rax), %%r9\n\t"                               \
+         "movq 40(%%rax), %%r8\n\t"                               \
+         "movq 32(%%rax), %%rcx\n\t"                              \
+         "movq 24(%%rax), %%rdx\n\t"                              \
+         "movq 16(%%rax), %%rsi\n\t"                              \
+         "movq 8(%%rax), %%rdi\n\t"                               \
+         "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
+         VALGRIND_CALL_NOREDIR_RAX                                \
+         "addq $32, %%rsp\n"                                      \
+         "addq $128,%%rsp\n\t"                                    \
+         : /*out*/   "=a" (_res)                                  \
+         : /*in*/    "a" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
+                                  arg7,arg8,arg9,arg10,arg11)     \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[12];                         \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)(arg1);                         \
+      _argvec[2] = (unsigned long)(arg2);                         \
+      _argvec[3] = (unsigned long)(arg3);                         \
+      _argvec[4] = (unsigned long)(arg4);                         \
+      _argvec[5] = (unsigned long)(arg5);                         \
+      _argvec[6] = (unsigned long)(arg6);                         \
+      _argvec[7] = (unsigned long)(arg7);                         \
+      _argvec[8] = (unsigned long)(arg8);                         \
+      _argvec[9] = (unsigned long)(arg9);                         \
+      _argvec[10] = (unsigned long)(arg10);                       \
+      _argvec[11] = (unsigned long)(arg11);                       \
+      __asm__ volatile(                                           \
+         "subq $128,%%rsp\n\t"                                    \
+         "pushq 88(%%rax)\n\t"                                    \
+         "pushq 80(%%rax)\n\t"                                    \
+         "pushq 72(%%rax)\n\t"                                    \
+         "pushq 64(%%rax)\n\t"                                    \
+         "pushq 56(%%rax)\n\t"                                    \
+         "movq 48(%%rax), %%r9\n\t"                               \
+         "movq 40(%%rax), %%r8\n\t"                               \
+         "movq 32(%%rax), %%rcx\n\t"                              \
+         "movq 24(%%rax), %%rdx\n\t"                              \
+         "movq 16(%%rax), %%rsi\n\t"                              \
+         "movq 8(%%rax), %%rdi\n\t"                               \
+         "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
+         VALGRIND_CALL_NOREDIR_RAX                                \
+         "addq $40, %%rsp\n"                                      \
+         "addq $128,%%rsp\n\t"                                    \
+         : /*out*/   "=a" (_res)                                  \
+         : /*in*/    "a" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
+                                arg7,arg8,arg9,arg10,arg11,arg12) \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[13];                         \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)(arg1);                         \
+      _argvec[2] = (unsigned long)(arg2);                         \
+      _argvec[3] = (unsigned long)(arg3);                         \
+      _argvec[4] = (unsigned long)(arg4);                         \
+      _argvec[5] = (unsigned long)(arg5);                         \
+      _argvec[6] = (unsigned long)(arg6);                         \
+      _argvec[7] = (unsigned long)(arg7);                         \
+      _argvec[8] = (unsigned long)(arg8);                         \
+      _argvec[9] = (unsigned long)(arg9);                         \
+      _argvec[10] = (unsigned long)(arg10);                       \
+      _argvec[11] = (unsigned long)(arg11);                       \
+      _argvec[12] = (unsigned long)(arg12);                       \
+      __asm__ volatile(                                           \
+         "subq $128,%%rsp\n\t"                                    \
+         "pushq 96(%%rax)\n\t"                                    \
+         "pushq 88(%%rax)\n\t"                                    \
+         "pushq 80(%%rax)\n\t"                                    \
+         "pushq 72(%%rax)\n\t"                                    \
+         "pushq 64(%%rax)\n\t"                                    \
+         "pushq 56(%%rax)\n\t"                                    \
+         "movq 48(%%rax), %%r9\n\t"                               \
+         "movq 40(%%rax), %%r8\n\t"                               \
+         "movq 32(%%rax), %%rcx\n\t"                              \
+         "movq 24(%%rax), %%rdx\n\t"                              \
+         "movq 16(%%rax), %%rsi\n\t"                              \
+         "movq 8(%%rax), %%rdi\n\t"                               \
+         "movq (%%rax), %%rax\n\t"  /* target->%rax */            \
+         VALGRIND_CALL_NOREDIR_RAX                                \
+         "addq $48, %%rsp\n"                                      \
+         "addq $128,%%rsp\n\t"                                    \
+         : /*out*/   "=a" (_res)                                  \
+         : /*in*/    "a" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#endif /* PLAT_amd64_linux */
+
+/* ------------------------ ppc32-linux ------------------------ */
+
+#if defined(PLAT_ppc32_linux)
+
+/* This is useful for finding out about the on-stack stuff:
+
+   extern int f9  ( int,int,int,int,int,int,int,int,int );
+   extern int f10 ( int,int,int,int,int,int,int,int,int,int );
+   extern int f11 ( int,int,int,int,int,int,int,int,int,int,int );
+   extern int f12 ( int,int,int,int,int,int,int,int,int,int,int,int );
+
+   int g9 ( void ) {
+      return f9(11,22,33,44,55,66,77,88,99);
+   }
+   int g10 ( void ) {
+      return f10(11,22,33,44,55,66,77,88,99,110);
+   }
+   int g11 ( void ) {
+      return f11(11,22,33,44,55,66,77,88,99,110,121);
+   }
+   int g12 ( void ) {
+      return f12(11,22,33,44,55,66,77,88,99,110,121,132);
+   }
+*/
+
+/* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
+
+/* These regs are trashed by the hidden call. */
+#define __CALLER_SAVED_REGS                                       \
+   "lr", "ctr", "xer",                                            \
+   "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
+   "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
+   "r11", "r12", "r13"
+
+/* These CALL_FN_ macros assume that on ppc32-linux, 
+   sizeof(unsigned long) == 4. */
+
+#define CALL_FN_W_v(lval, orig)                                   \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[1];                          \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         "lwz 11,0(11)\n\t"  /* target->r11 */                    \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr %0,3"                                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_W(lval, orig, arg1)                             \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[2];                          \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)arg1;                           \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
+         "lwz 11,0(11)\n\t"  /* target->r11 */                    \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr %0,3"                                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3];                          \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)arg1;                           \
+      _argvec[2] = (unsigned long)arg2;                           \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
+         "lwz 4,8(11)\n\t"                                        \
+         "lwz 11,0(11)\n\t"  /* target->r11 */                    \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr %0,3"                                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[4];                          \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)arg1;                           \
+      _argvec[2] = (unsigned long)arg2;                           \
+      _argvec[3] = (unsigned long)arg3;                           \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
+         "lwz 4,8(11)\n\t"                                        \
+         "lwz 5,12(11)\n\t"                                       \
+         "lwz 11,0(11)\n\t"  /* target->r11 */                    \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr %0,3"                                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[5];                          \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)arg1;                           \
+      _argvec[2] = (unsigned long)arg2;                           \
+      _argvec[3] = (unsigned long)arg3;                           \
+      _argvec[4] = (unsigned long)arg4;                           \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
+         "lwz 4,8(11)\n\t"                                        \
+         "lwz 5,12(11)\n\t"                                       \
+         "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
+         "lwz 11,0(11)\n\t"  /* target->r11 */                    \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr %0,3"                                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[6];                          \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)arg1;                           \
+      _argvec[2] = (unsigned long)arg2;                           \
+      _argvec[3] = (unsigned long)arg3;                           \
+      _argvec[4] = (unsigned long)arg4;                           \
+      _argvec[5] = (unsigned long)arg5;                           \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
+         "lwz 4,8(11)\n\t"                                        \
+         "lwz 5,12(11)\n\t"                                       \
+         "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
+         "lwz 7,20(11)\n\t"                                       \
+         "lwz 11,0(11)\n\t"  /* target->r11 */                    \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr %0,3"                                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[7];                          \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)arg1;                           \
+      _argvec[2] = (unsigned long)arg2;                           \
+      _argvec[3] = (unsigned long)arg3;                           \
+      _argvec[4] = (unsigned long)arg4;                           \
+      _argvec[5] = (unsigned long)arg5;                           \
+      _argvec[6] = (unsigned long)arg6;                           \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
+         "lwz 4,8(11)\n\t"                                        \
+         "lwz 5,12(11)\n\t"                                       \
+         "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
+         "lwz 7,20(11)\n\t"                                       \
+         "lwz 8,24(11)\n\t"                                       \
+         "lwz 11,0(11)\n\t"  /* target->r11 */                    \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr %0,3"                                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
+                                 arg7)                            \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[8];                          \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)arg1;                           \
+      _argvec[2] = (unsigned long)arg2;                           \
+      _argvec[3] = (unsigned long)arg3;                           \
+      _argvec[4] = (unsigned long)arg4;                           \
+      _argvec[5] = (unsigned long)arg5;                           \
+      _argvec[6] = (unsigned long)arg6;                           \
+      _argvec[7] = (unsigned long)arg7;                           \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
+         "lwz 4,8(11)\n\t"                                        \
+         "lwz 5,12(11)\n\t"                                       \
+         "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
+         "lwz 7,20(11)\n\t"                                       \
+         "lwz 8,24(11)\n\t"                                       \
+         "lwz 9,28(11)\n\t"                                       \
+         "lwz 11,0(11)\n\t"  /* target->r11 */                    \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr %0,3"                                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
+                                 arg7,arg8)                       \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[9];                          \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)arg1;                           \
+      _argvec[2] = (unsigned long)arg2;                           \
+      _argvec[3] = (unsigned long)arg3;                           \
+      _argvec[4] = (unsigned long)arg4;                           \
+      _argvec[5] = (unsigned long)arg5;                           \
+      _argvec[6] = (unsigned long)arg6;                           \
+      _argvec[7] = (unsigned long)arg7;                           \
+      _argvec[8] = (unsigned long)arg8;                           \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
+         "lwz 4,8(11)\n\t"                                        \
+         "lwz 5,12(11)\n\t"                                       \
+         "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
+         "lwz 7,20(11)\n\t"                                       \
+         "lwz 8,24(11)\n\t"                                       \
+         "lwz 9,28(11)\n\t"                                       \
+         "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
+         "lwz 11,0(11)\n\t"  /* target->r11 */                    \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr %0,3"                                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
+                                 arg7,arg8,arg9)                  \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[10];                         \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)arg1;                           \
+      _argvec[2] = (unsigned long)arg2;                           \
+      _argvec[3] = (unsigned long)arg3;                           \
+      _argvec[4] = (unsigned long)arg4;                           \
+      _argvec[5] = (unsigned long)arg5;                           \
+      _argvec[6] = (unsigned long)arg6;                           \
+      _argvec[7] = (unsigned long)arg7;                           \
+      _argvec[8] = (unsigned long)arg8;                           \
+      _argvec[9] = (unsigned long)arg9;                           \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         "addi 1,1,-16\n\t"                                       \
+         /* arg9 */                                               \
+         "lwz 3,36(11)\n\t"                                       \
+         "stw 3,8(1)\n\t"                                         \
+         /* args1-8 */                                            \
+         "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
+         "lwz 4,8(11)\n\t"                                        \
+         "lwz 5,12(11)\n\t"                                       \
+         "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
+         "lwz 7,20(11)\n\t"                                       \
+         "lwz 8,24(11)\n\t"                                       \
+         "lwz 9,28(11)\n\t"                                       \
+         "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
+         "lwz 11,0(11)\n\t"  /* target->r11 */                    \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "addi 1,1,16\n\t"                                        \
+         "mr %0,3"                                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
+                                  arg7,arg8,arg9,arg10)           \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[11];                         \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)arg1;                           \
+      _argvec[2] = (unsigned long)arg2;                           \
+      _argvec[3] = (unsigned long)arg3;                           \
+      _argvec[4] = (unsigned long)arg4;                           \
+      _argvec[5] = (unsigned long)arg5;                           \
+      _argvec[6] = (unsigned long)arg6;                           \
+      _argvec[7] = (unsigned long)arg7;                           \
+      _argvec[8] = (unsigned long)arg8;                           \
+      _argvec[9] = (unsigned long)arg9;                           \
+      _argvec[10] = (unsigned long)arg10;                         \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         "addi 1,1,-16\n\t"                                       \
+         /* arg10 */                                              \
+         "lwz 3,40(11)\n\t"                                       \
+         "stw 3,12(1)\n\t"                                        \
+         /* arg9 */                                               \
+         "lwz 3,36(11)\n\t"                                       \
+         "stw 3,8(1)\n\t"                                         \
+         /* args1-8 */                                            \
+         "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
+         "lwz 4,8(11)\n\t"                                        \
+         "lwz 5,12(11)\n\t"                                       \
+         "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
+         "lwz 7,20(11)\n\t"                                       \
+         "lwz 8,24(11)\n\t"                                       \
+         "lwz 9,28(11)\n\t"                                       \
+         "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
+         "lwz 11,0(11)\n\t"  /* target->r11 */                    \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "addi 1,1,16\n\t"                                        \
+         "mr %0,3"                                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
+                                  arg7,arg8,arg9,arg10,arg11)     \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[12];                         \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)arg1;                           \
+      _argvec[2] = (unsigned long)arg2;                           \
+      _argvec[3] = (unsigned long)arg3;                           \
+      _argvec[4] = (unsigned long)arg4;                           \
+      _argvec[5] = (unsigned long)arg5;                           \
+      _argvec[6] = (unsigned long)arg6;                           \
+      _argvec[7] = (unsigned long)arg7;                           \
+      _argvec[8] = (unsigned long)arg8;                           \
+      _argvec[9] = (unsigned long)arg9;                           \
+      _argvec[10] = (unsigned long)arg10;                         \
+      _argvec[11] = (unsigned long)arg11;                         \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         "addi 1,1,-32\n\t"                                       \
+         /* arg11 */                                              \
+         "lwz 3,44(11)\n\t"                                       \
+         "stw 3,16(1)\n\t"                                        \
+         /* arg10 */                                              \
+         "lwz 3,40(11)\n\t"                                       \
+         "stw 3,12(1)\n\t"                                        \
+         /* arg9 */                                               \
+         "lwz 3,36(11)\n\t"                                       \
+         "stw 3,8(1)\n\t"                                         \
+         /* args1-8 */                                            \
+         "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
+         "lwz 4,8(11)\n\t"                                        \
+         "lwz 5,12(11)\n\t"                                       \
+         "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
+         "lwz 7,20(11)\n\t"                                       \
+         "lwz 8,24(11)\n\t"                                       \
+         "lwz 9,28(11)\n\t"                                       \
+         "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
+         "lwz 11,0(11)\n\t"  /* target->r11 */                    \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "addi 1,1,32\n\t"                                        \
+         "mr %0,3"                                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
+                                arg7,arg8,arg9,arg10,arg11,arg12) \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[13];                         \
+      volatile unsigned long _res;                                \
+      _argvec[0] = (unsigned long)_orig.nraddr;                   \
+      _argvec[1] = (unsigned long)arg1;                           \
+      _argvec[2] = (unsigned long)arg2;                           \
+      _argvec[3] = (unsigned long)arg3;                           \
+      _argvec[4] = (unsigned long)arg4;                           \
+      _argvec[5] = (unsigned long)arg5;                           \
+      _argvec[6] = (unsigned long)arg6;                           \
+      _argvec[7] = (unsigned long)arg7;                           \
+      _argvec[8] = (unsigned long)arg8;                           \
+      _argvec[9] = (unsigned long)arg9;                           \
+      _argvec[10] = (unsigned long)arg10;                         \
+      _argvec[11] = (unsigned long)arg11;                         \
+      _argvec[12] = (unsigned long)arg12;                         \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         "addi 1,1,-32\n\t"                                       \
+         /* arg12 */                                              \
+         "lwz 3,48(11)\n\t"                                       \
+         "stw 3,20(1)\n\t"                                        \
+         /* arg11 */                                              \
+         "lwz 3,44(11)\n\t"                                       \
+         "stw 3,16(1)\n\t"                                        \
+         /* arg10 */                                              \
+         "lwz 3,40(11)\n\t"                                       \
+         "stw 3,12(1)\n\t"                                        \
+         /* arg9 */                                               \
+         "lwz 3,36(11)\n\t"                                       \
+         "stw 3,8(1)\n\t"                                         \
+         /* args1-8 */                                            \
+         "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
+         "lwz 4,8(11)\n\t"                                        \
+         "lwz 5,12(11)\n\t"                                       \
+         "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
+         "lwz 7,20(11)\n\t"                                       \
+         "lwz 8,24(11)\n\t"                                       \
+         "lwz 9,28(11)\n\t"                                       \
+         "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
+         "lwz 11,0(11)\n\t"  /* target->r11 */                    \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "addi 1,1,32\n\t"                                        \
+         "mr %0,3"                                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[0])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#endif /* PLAT_ppc32_linux */
+
+/* ------------------------ ppc64-linux ------------------------ */
+
+#if defined(PLAT_ppc64_linux)
+
+/* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
+
+/* These regs are trashed by the hidden call. */
+#define __CALLER_SAVED_REGS                                       \
+   "lr", "ctr", "xer",                                            \
+   "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
+   "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
+   "r11", "r12", "r13"
+
+/* These CALL_FN_ macros assume that on ppc64-linux, sizeof(unsigned
+   long) == 8. */
+
+#define CALL_FN_W_v(lval, orig)                                   \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+0];                        \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1] = (unsigned long)_orig.r2;                       \
+      _argvec[2] = (unsigned long)_orig.nraddr;                   \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         "std 2,-16(11)\n\t"  /* save tocptr */                   \
+         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
+         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "ld 2,-16(11)" /* restore tocptr */                      \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_W(lval, orig, arg1)                             \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+1];                        \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         "std 2,-16(11)\n\t"  /* save tocptr */                   \
+         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
+         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
+         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "ld 2,-16(11)" /* restore tocptr */                      \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+2];                        \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      _argvec[2+2] = (unsigned long)arg2;                         \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         "std 2,-16(11)\n\t"  /* save tocptr */                   \
+         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
+         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
+         "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
+         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "ld 2,-16(11)" /* restore tocptr */                      \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+3];                        \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      _argvec[2+2] = (unsigned long)arg2;                         \
+      _argvec[2+3] = (unsigned long)arg3;                         \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         "std 2,-16(11)\n\t"  /* save tocptr */                   \
+         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
+         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
+         "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
+         "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
+         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "ld 2,-16(11)" /* restore tocptr */                      \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+4];                        \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      _argvec[2+2] = (unsigned long)arg2;                         \
+      _argvec[2+3] = (unsigned long)arg3;                         \
+      _argvec[2+4] = (unsigned long)arg4;                         \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         "std 2,-16(11)\n\t"  /* save tocptr */                   \
+         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
+         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
+         "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
+         "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
+         "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
+         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "ld 2,-16(11)" /* restore tocptr */                      \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+5];                        \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      _argvec[2+2] = (unsigned long)arg2;                         \
+      _argvec[2+3] = (unsigned long)arg3;                         \
+      _argvec[2+4] = (unsigned long)arg4;                         \
+      _argvec[2+5] = (unsigned long)arg5;                         \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         "std 2,-16(11)\n\t"  /* save tocptr */                   \
+         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
+         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
+         "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
+         "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
+         "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
+         "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
+         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "ld 2,-16(11)" /* restore tocptr */                      \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+6];                        \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      _argvec[2+2] = (unsigned long)arg2;                         \
+      _argvec[2+3] = (unsigned long)arg3;                         \
+      _argvec[2+4] = (unsigned long)arg4;                         \
+      _argvec[2+5] = (unsigned long)arg5;                         \
+      _argvec[2+6] = (unsigned long)arg6;                         \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         "std 2,-16(11)\n\t"  /* save tocptr */                   \
+         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
+         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
+         "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
+         "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
+         "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
+         "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
+         "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
+         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "ld 2,-16(11)" /* restore tocptr */                      \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
+                                 arg7)                            \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+7];                        \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      _argvec[2+2] = (unsigned long)arg2;                         \
+      _argvec[2+3] = (unsigned long)arg3;                         \
+      _argvec[2+4] = (unsigned long)arg4;                         \
+      _argvec[2+5] = (unsigned long)arg5;                         \
+      _argvec[2+6] = (unsigned long)arg6;                         \
+      _argvec[2+7] = (unsigned long)arg7;                         \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         "std 2,-16(11)\n\t"  /* save tocptr */                   \
+         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
+         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
+         "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
+         "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
+         "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
+         "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
+         "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
+         "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
+         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "ld 2,-16(11)" /* restore tocptr */                      \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
+                                 arg7,arg8)                       \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+8];                        \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      _argvec[2+2] = (unsigned long)arg2;                         \
+      _argvec[2+3] = (unsigned long)arg3;                         \
+      _argvec[2+4] = (unsigned long)arg4;                         \
+      _argvec[2+5] = (unsigned long)arg5;                         \
+      _argvec[2+6] = (unsigned long)arg6;                         \
+      _argvec[2+7] = (unsigned long)arg7;                         \
+      _argvec[2+8] = (unsigned long)arg8;                         \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         "std 2,-16(11)\n\t"  /* save tocptr */                   \
+         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
+         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
+         "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
+         "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
+         "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
+         "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
+         "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
+         "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
+         "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
+         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "ld 2,-16(11)" /* restore tocptr */                      \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
+                                 arg7,arg8,arg9)                  \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+9];                        \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      _argvec[2+2] = (unsigned long)arg2;                         \
+      _argvec[2+3] = (unsigned long)arg3;                         \
+      _argvec[2+4] = (unsigned long)arg4;                         \
+      _argvec[2+5] = (unsigned long)arg5;                         \
+      _argvec[2+6] = (unsigned long)arg6;                         \
+      _argvec[2+7] = (unsigned long)arg7;                         \
+      _argvec[2+8] = (unsigned long)arg8;                         \
+      _argvec[2+9] = (unsigned long)arg9;                         \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         "std 2,-16(11)\n\t"  /* save tocptr */                   \
+         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
+         "addi 1,1,-128\n\t"  /* expand stack frame */            \
+         /* arg9 */                                               \
+         "ld  3,72(11)\n\t"                                       \
+         "std 3,112(1)\n\t"                                       \
+         /* args1-8 */                                            \
+         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
+         "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
+         "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
+         "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
+         "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
+         "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
+         "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
+         "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
+         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "ld 2,-16(11)\n\t" /* restore tocptr */                  \
+         "addi 1,1,128"     /* restore frame */                   \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
+                                  arg7,arg8,arg9,arg10)           \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+10];                       \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      _argvec[2+2] = (unsigned long)arg2;                         \
+      _argvec[2+3] = (unsigned long)arg3;                         \
+      _argvec[2+4] = (unsigned long)arg4;                         \
+      _argvec[2+5] = (unsigned long)arg5;                         \
+      _argvec[2+6] = (unsigned long)arg6;                         \
+      _argvec[2+7] = (unsigned long)arg7;                         \
+      _argvec[2+8] = (unsigned long)arg8;                         \
+      _argvec[2+9] = (unsigned long)arg9;                         \
+      _argvec[2+10] = (unsigned long)arg10;                       \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         "std 2,-16(11)\n\t"  /* save tocptr */                   \
+         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
+         "addi 1,1,-128\n\t"  /* expand stack frame */            \
+         /* arg10 */                                              \
+         "ld  3,80(11)\n\t"                                       \
+         "std 3,120(1)\n\t"                                       \
+         /* arg9 */                                               \
+         "ld  3,72(11)\n\t"                                       \
+         "std 3,112(1)\n\t"                                       \
+         /* args1-8 */                                            \
+         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
+         "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
+         "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
+         "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
+         "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
+         "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
+         "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
+         "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
+         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "ld 2,-16(11)\n\t" /* restore tocptr */                  \
+         "addi 1,1,128"     /* restore frame */                   \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
+                                  arg7,arg8,arg9,arg10,arg11)     \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+11];                       \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      _argvec[2+2] = (unsigned long)arg2;                         \
+      _argvec[2+3] = (unsigned long)arg3;                         \
+      _argvec[2+4] = (unsigned long)arg4;                         \
+      _argvec[2+5] = (unsigned long)arg5;                         \
+      _argvec[2+6] = (unsigned long)arg6;                         \
+      _argvec[2+7] = (unsigned long)arg7;                         \
+      _argvec[2+8] = (unsigned long)arg8;                         \
+      _argvec[2+9] = (unsigned long)arg9;                         \
+      _argvec[2+10] = (unsigned long)arg10;                       \
+      _argvec[2+11] = (unsigned long)arg11;                       \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         "std 2,-16(11)\n\t"  /* save tocptr */                   \
+         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
+         "addi 1,1,-144\n\t"  /* expand stack frame */            \
+         /* arg11 */                                              \
+         "ld  3,88(11)\n\t"                                       \
+         "std 3,128(1)\n\t"                                       \
+         /* arg10 */                                              \
+         "ld  3,80(11)\n\t"                                       \
+         "std 3,120(1)\n\t"                                       \
+         /* arg9 */                                               \
+         "ld  3,72(11)\n\t"                                       \
+         "std 3,112(1)\n\t"                                       \
+         /* args1-8 */                                            \
+         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
+         "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
+         "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
+         "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
+         "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
+         "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
+         "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
+         "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
+         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "ld 2,-16(11)\n\t" /* restore tocptr */                  \
+         "addi 1,1,144"     /* restore frame */                   \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
+                                arg7,arg8,arg9,arg10,arg11,arg12) \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+12];                       \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      _argvec[2+2] = (unsigned long)arg2;                         \
+      _argvec[2+3] = (unsigned long)arg3;                         \
+      _argvec[2+4] = (unsigned long)arg4;                         \
+      _argvec[2+5] = (unsigned long)arg5;                         \
+      _argvec[2+6] = (unsigned long)arg6;                         \
+      _argvec[2+7] = (unsigned long)arg7;                         \
+      _argvec[2+8] = (unsigned long)arg8;                         \
+      _argvec[2+9] = (unsigned long)arg9;                         \
+      _argvec[2+10] = (unsigned long)arg10;                       \
+      _argvec[2+11] = (unsigned long)arg11;                       \
+      _argvec[2+12] = (unsigned long)arg12;                       \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         "std 2,-16(11)\n\t"  /* save tocptr */                   \
+         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
+         "addi 1,1,-144\n\t"  /* expand stack frame */            \
+         /* arg12 */                                              \
+         "ld  3,96(11)\n\t"                                       \
+         "std 3,136(1)\n\t"                                       \
+         /* arg11 */                                              \
+         "ld  3,88(11)\n\t"                                       \
+         "std 3,128(1)\n\t"                                       \
+         /* arg10 */                                              \
+         "ld  3,80(11)\n\t"                                       \
+         "std 3,120(1)\n\t"                                       \
+         /* arg9 */                                               \
+         "ld  3,72(11)\n\t"                                       \
+         "std 3,112(1)\n\t"                                       \
+         /* args1-8 */                                            \
+         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
+         "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
+         "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
+         "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
+         "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
+         "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
+         "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
+         "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
+         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "ld 2,-16(11)\n\t" /* restore tocptr */                  \
+         "addi 1,1,144"     /* restore frame */                   \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#endif /* PLAT_ppc64_linux */
+
+/* ------------------------ ppc32-aix5 ------------------------- */
+
+#if defined(PLAT_ppc32_aix5)
+
+/* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
+
+/* These regs are trashed by the hidden call. */
+#define __CALLER_SAVED_REGS                                       \
+   "lr", "ctr", "xer",                                            \
+   "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
+   "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
+   "r11", "r12", "r13"
+
+/* Expand the stack frame, copying enough info that unwinding
+   still works.  Trashes r3. */
+
+#define VG_EXPAND_FRAME_BY_trashes_r3(_n_fr)                      \
+         "addi 1,1,-" #_n_fr "\n\t"                               \
+         "lwz  3," #_n_fr "(1)\n\t"                               \
+         "stw  3,0(1)\n\t"
+
+#define VG_CONTRACT_FRAME_BY(_n_fr)                               \
+         "addi 1,1," #_n_fr "\n\t"
+
+/* These CALL_FN_ macros assume that on ppc32-aix5, sizeof(unsigned
+   long) == 4. */
+
+#define CALL_FN_W_v(lval, orig)                                   \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+0];                        \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1] = (unsigned long)_orig.r2;                       \
+      _argvec[2] = (unsigned long)_orig.nraddr;                   \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
+         "stw  2,-8(11)\n\t"  /* save tocptr */                   \
+         "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
+         "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
+         VG_CONTRACT_FRAME_BY(512)                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_W(lval, orig, arg1)                             \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+1];                        \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
+         "stw  2,-8(11)\n\t"  /* save tocptr */                   \
+         "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
+         "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
+         "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
+         VG_CONTRACT_FRAME_BY(512)                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+2];                        \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      _argvec[2+2] = (unsigned long)arg2;                         \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
+         "stw  2,-8(11)\n\t"  /* save tocptr */                   \
+         "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
+         "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
+         "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
+         "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
+         VG_CONTRACT_FRAME_BY(512)                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+3];                        \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      _argvec[2+2] = (unsigned long)arg2;                         \
+      _argvec[2+3] = (unsigned long)arg3;                         \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
+         "stw  2,-8(11)\n\t"  /* save tocptr */                   \
+         "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
+         "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
+         "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
+         "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
+         "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
+         VG_CONTRACT_FRAME_BY(512)                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+4];                        \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      _argvec[2+2] = (unsigned long)arg2;                         \
+      _argvec[2+3] = (unsigned long)arg3;                         \
+      _argvec[2+4] = (unsigned long)arg4;                         \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
+         "stw  2,-8(11)\n\t"  /* save tocptr */                   \
+         "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
+         "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
+         "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
+         "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
+         "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
+         "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
+         VG_CONTRACT_FRAME_BY(512)                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+5];                        \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      _argvec[2+2] = (unsigned long)arg2;                         \
+      _argvec[2+3] = (unsigned long)arg3;                         \
+      _argvec[2+4] = (unsigned long)arg4;                         \
+      _argvec[2+5] = (unsigned long)arg5;                         \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
+         "stw  2,-8(11)\n\t"  /* save tocptr */                   \
+         "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
+         "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
+         "lwz  4, 8(11)\n\t" /* arg2->r4 */                       \
+         "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
+         "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
+         "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
+         "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
+         VG_CONTRACT_FRAME_BY(512)                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+6];                        \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      _argvec[2+2] = (unsigned long)arg2;                         \
+      _argvec[2+3] = (unsigned long)arg3;                         \
+      _argvec[2+4] = (unsigned long)arg4;                         \
+      _argvec[2+5] = (unsigned long)arg5;                         \
+      _argvec[2+6] = (unsigned long)arg6;                         \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
+         "stw  2,-8(11)\n\t"  /* save tocptr */                   \
+         "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
+         "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
+         "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
+         "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
+         "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
+         "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
+         "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
+         "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
+         VG_CONTRACT_FRAME_BY(512)                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
+                                 arg7)                            \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+7];                        \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      _argvec[2+2] = (unsigned long)arg2;                         \
+      _argvec[2+3] = (unsigned long)arg3;                         \
+      _argvec[2+4] = (unsigned long)arg4;                         \
+      _argvec[2+5] = (unsigned long)arg5;                         \
+      _argvec[2+6] = (unsigned long)arg6;                         \
+      _argvec[2+7] = (unsigned long)arg7;                         \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
+         "stw  2,-8(11)\n\t"  /* save tocptr */                   \
+         "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
+         "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
+         "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
+         "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
+         "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
+         "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
+         "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
+         "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
+         "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
+         VG_CONTRACT_FRAME_BY(512)                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
+                                 arg7,arg8)                       \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+8];                        \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      _argvec[2+2] = (unsigned long)arg2;                         \
+      _argvec[2+3] = (unsigned long)arg3;                         \
+      _argvec[2+4] = (unsigned long)arg4;                         \
+      _argvec[2+5] = (unsigned long)arg5;                         \
+      _argvec[2+6] = (unsigned long)arg6;                         \
+      _argvec[2+7] = (unsigned long)arg7;                         \
+      _argvec[2+8] = (unsigned long)arg8;                         \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
+         "stw  2,-8(11)\n\t"  /* save tocptr */                   \
+         "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
+         "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
+         "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
+         "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
+         "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
+         "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
+         "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
+         "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
+         "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
+         "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
+         VG_CONTRACT_FRAME_BY(512)                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
+                                 arg7,arg8,arg9)                  \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+9];                        \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      _argvec[2+2] = (unsigned long)arg2;                         \
+      _argvec[2+3] = (unsigned long)arg3;                         \
+      _argvec[2+4] = (unsigned long)arg4;                         \
+      _argvec[2+5] = (unsigned long)arg5;                         \
+      _argvec[2+6] = (unsigned long)arg6;                         \
+      _argvec[2+7] = (unsigned long)arg7;                         \
+      _argvec[2+8] = (unsigned long)arg8;                         \
+      _argvec[2+9] = (unsigned long)arg9;                         \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
+         "stw  2,-8(11)\n\t"  /* save tocptr */                   \
+         "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
+         VG_EXPAND_FRAME_BY_trashes_r3(64)                        \
+         /* arg9 */                                               \
+         "lwz 3,36(11)\n\t"                                       \
+         "stw 3,56(1)\n\t"                                        \
+         /* args1-8 */                                            \
+         "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
+         "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
+         "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
+         "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
+         "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
+         "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
+         "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
+         "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
+         "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
+         VG_CONTRACT_FRAME_BY(64)                                 \
+         VG_CONTRACT_FRAME_BY(512)                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
+                                  arg7,arg8,arg9,arg10)           \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+10];                       \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      _argvec[2+2] = (unsigned long)arg2;                         \
+      _argvec[2+3] = (unsigned long)arg3;                         \
+      _argvec[2+4] = (unsigned long)arg4;                         \
+      _argvec[2+5] = (unsigned long)arg5;                         \
+      _argvec[2+6] = (unsigned long)arg6;                         \
+      _argvec[2+7] = (unsigned long)arg7;                         \
+      _argvec[2+8] = (unsigned long)arg8;                         \
+      _argvec[2+9] = (unsigned long)arg9;                         \
+      _argvec[2+10] = (unsigned long)arg10;                       \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
+         "stw  2,-8(11)\n\t"  /* save tocptr */                   \
+         "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
+         VG_EXPAND_FRAME_BY_trashes_r3(64)                        \
+         /* arg10 */                                              \
+         "lwz 3,40(11)\n\t"                                       \
+         "stw 3,60(1)\n\t"                                        \
+         /* arg9 */                                               \
+         "lwz 3,36(11)\n\t"                                       \
+         "stw 3,56(1)\n\t"                                        \
+         /* args1-8 */                                            \
+         "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
+         "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
+         "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
+         "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
+         "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
+         "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
+         "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
+         "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
+         "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
+         VG_CONTRACT_FRAME_BY(64)                                 \
+         VG_CONTRACT_FRAME_BY(512)                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
+                                  arg7,arg8,arg9,arg10,arg11)     \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+11];                       \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      _argvec[2+2] = (unsigned long)arg2;                         \
+      _argvec[2+3] = (unsigned long)arg3;                         \
+      _argvec[2+4] = (unsigned long)arg4;                         \
+      _argvec[2+5] = (unsigned long)arg5;                         \
+      _argvec[2+6] = (unsigned long)arg6;                         \
+      _argvec[2+7] = (unsigned long)arg7;                         \
+      _argvec[2+8] = (unsigned long)arg8;                         \
+      _argvec[2+9] = (unsigned long)arg9;                         \
+      _argvec[2+10] = (unsigned long)arg10;                       \
+      _argvec[2+11] = (unsigned long)arg11;                       \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
+         "stw  2,-8(11)\n\t"  /* save tocptr */                   \
+         "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
+         VG_EXPAND_FRAME_BY_trashes_r3(72)                        \
+         /* arg11 */                                              \
+         "lwz 3,44(11)\n\t"                                       \
+         "stw 3,64(1)\n\t"                                        \
+         /* arg10 */                                              \
+         "lwz 3,40(11)\n\t"                                       \
+         "stw 3,60(1)\n\t"                                        \
+         /* arg9 */                                               \
+         "lwz 3,36(11)\n\t"                                       \
+         "stw 3,56(1)\n\t"                                        \
+         /* args1-8 */                                            \
+         "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
+         "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
+         "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
+         "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
+         "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
+         "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
+         "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
+         "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
+         "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
+         VG_CONTRACT_FRAME_BY(72)                                 \
+         VG_CONTRACT_FRAME_BY(512)                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
+                                arg7,arg8,arg9,arg10,arg11,arg12) \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+12];                       \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      _argvec[2+2] = (unsigned long)arg2;                         \
+      _argvec[2+3] = (unsigned long)arg3;                         \
+      _argvec[2+4] = (unsigned long)arg4;                         \
+      _argvec[2+5] = (unsigned long)arg5;                         \
+      _argvec[2+6] = (unsigned long)arg6;                         \
+      _argvec[2+7] = (unsigned long)arg7;                         \
+      _argvec[2+8] = (unsigned long)arg8;                         \
+      _argvec[2+9] = (unsigned long)arg9;                         \
+      _argvec[2+10] = (unsigned long)arg10;                       \
+      _argvec[2+11] = (unsigned long)arg11;                       \
+      _argvec[2+12] = (unsigned long)arg12;                       \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
+         "stw  2,-8(11)\n\t"  /* save tocptr */                   \
+         "lwz  2,-4(11)\n\t"  /* use nraddr's tocptr */           \
+         VG_EXPAND_FRAME_BY_trashes_r3(72)                        \
+         /* arg12 */                                              \
+         "lwz 3,48(11)\n\t"                                       \
+         "stw 3,68(1)\n\t"                                        \
+         /* arg11 */                                              \
+         "lwz 3,44(11)\n\t"                                       \
+         "stw 3,64(1)\n\t"                                        \
+         /* arg10 */                                              \
+         "lwz 3,40(11)\n\t"                                       \
+         "stw 3,60(1)\n\t"                                        \
+         /* arg9 */                                               \
+         "lwz 3,36(11)\n\t"                                       \
+         "stw 3,56(1)\n\t"                                        \
+         /* args1-8 */                                            \
+         "lwz  3, 4(11)\n\t"  /* arg1->r3 */                      \
+         "lwz  4, 8(11)\n\t"  /* arg2->r4 */                      \
+         "lwz  5, 12(11)\n\t" /* arg3->r5 */                      \
+         "lwz  6, 16(11)\n\t" /* arg4->r6 */                      \
+         "lwz  7, 20(11)\n\t" /* arg5->r7 */                      \
+         "lwz  8, 24(11)\n\t" /* arg6->r8 */                      \
+         "lwz  9, 28(11)\n\t" /* arg7->r9 */                      \
+         "lwz 10, 32(11)\n\t" /* arg8->r10 */                     \
+         "lwz 11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "lwz 2,-8(11)\n\t" /* restore tocptr */                  \
+         VG_CONTRACT_FRAME_BY(72)                                 \
+         VG_CONTRACT_FRAME_BY(512)                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#endif /* PLAT_ppc32_aix5 */
+
+/* ------------------------ ppc64-aix5 ------------------------- */
+
+#if defined(PLAT_ppc64_aix5)
+
+/* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
+
+/* These regs are trashed by the hidden call. */
+#define __CALLER_SAVED_REGS                                       \
+   "lr", "ctr", "xer",                                            \
+   "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
+   "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
+   "r11", "r12", "r13"
+
+/* Expand the stack frame, copying enough info that unwinding
+   still works.  Trashes r3. */
+
+#define VG_EXPAND_FRAME_BY_trashes_r3(_n_fr)                      \
+         "addi 1,1,-" #_n_fr "\n\t"                               \
+         "ld   3," #_n_fr "(1)\n\t"                               \
+         "std  3,0(1)\n\t"
+
+#define VG_CONTRACT_FRAME_BY(_n_fr)                               \
+         "addi 1,1," #_n_fr "\n\t"
+
+/* These CALL_FN_ macros assume that on ppc64-aix5, sizeof(unsigned
+   long) == 8. */
+
+#define CALL_FN_W_v(lval, orig)                                   \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+0];                        \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1] = (unsigned long)_orig.r2;                       \
+      _argvec[2] = (unsigned long)_orig.nraddr;                   \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
+         "std  2,-16(11)\n\t" /* save tocptr */                   \
+         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
+         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "ld 2,-16(11)\n\t" /* restore tocptr */                  \
+         VG_CONTRACT_FRAME_BY(512)                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_W(lval, orig, arg1)                             \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+1];                        \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
+         "std  2,-16(11)\n\t" /* save tocptr */                   \
+         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
+         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
+         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "ld 2,-16(11)\n\t" /* restore tocptr */                  \
+         VG_CONTRACT_FRAME_BY(512)                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+2];                        \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      _argvec[2+2] = (unsigned long)arg2;                         \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
+         "std  2,-16(11)\n\t" /* save tocptr */                   \
+         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
+         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
+         "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
+         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "ld  2,-16(11)\n\t" /* restore tocptr */                 \
+         VG_CONTRACT_FRAME_BY(512)                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+3];                        \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      _argvec[2+2] = (unsigned long)arg2;                         \
+      _argvec[2+3] = (unsigned long)arg3;                         \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
+         "std  2,-16(11)\n\t" /* save tocptr */                   \
+         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
+         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
+         "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
+         "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
+         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "ld  2,-16(11)\n\t" /* restore tocptr */                 \
+         VG_CONTRACT_FRAME_BY(512)                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+4];                        \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      _argvec[2+2] = (unsigned long)arg2;                         \
+      _argvec[2+3] = (unsigned long)arg3;                         \
+      _argvec[2+4] = (unsigned long)arg4;                         \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
+         "std  2,-16(11)\n\t" /* save tocptr */                   \
+         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
+         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
+         "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
+         "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
+         "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
+         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "ld  2,-16(11)\n\t" /* restore tocptr */                 \
+         VG_CONTRACT_FRAME_BY(512)                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+5];                        \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      _argvec[2+2] = (unsigned long)arg2;                         \
+      _argvec[2+3] = (unsigned long)arg3;                         \
+      _argvec[2+4] = (unsigned long)arg4;                         \
+      _argvec[2+5] = (unsigned long)arg5;                         \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
+         "std  2,-16(11)\n\t" /* save tocptr */                   \
+         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
+         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
+         "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
+         "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
+         "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
+         "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
+         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "ld  2,-16(11)\n\t" /* restore tocptr */                 \
+         VG_CONTRACT_FRAME_BY(512)                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+6];                        \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      _argvec[2+2] = (unsigned long)arg2;                         \
+      _argvec[2+3] = (unsigned long)arg3;                         \
+      _argvec[2+4] = (unsigned long)arg4;                         \
+      _argvec[2+5] = (unsigned long)arg5;                         \
+      _argvec[2+6] = (unsigned long)arg6;                         \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
+         "std  2,-16(11)\n\t" /* save tocptr */                   \
+         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
+         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
+         "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
+         "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
+         "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
+         "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
+         "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
+         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "ld  2,-16(11)\n\t" /* restore tocptr */                 \
+         VG_CONTRACT_FRAME_BY(512)                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
+                                 arg7)                            \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+7];                        \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      _argvec[2+2] = (unsigned long)arg2;                         \
+      _argvec[2+3] = (unsigned long)arg3;                         \
+      _argvec[2+4] = (unsigned long)arg4;                         \
+      _argvec[2+5] = (unsigned long)arg5;                         \
+      _argvec[2+6] = (unsigned long)arg6;                         \
+      _argvec[2+7] = (unsigned long)arg7;                         \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
+         "std  2,-16(11)\n\t" /* save tocptr */                   \
+         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
+         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
+         "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
+         "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
+         "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
+         "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
+         "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
+         "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
+         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "ld  2,-16(11)\n\t" /* restore tocptr */                 \
+         VG_CONTRACT_FRAME_BY(512)                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
+                                 arg7,arg8)                       \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+8];                        \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      _argvec[2+2] = (unsigned long)arg2;                         \
+      _argvec[2+3] = (unsigned long)arg3;                         \
+      _argvec[2+4] = (unsigned long)arg4;                         \
+      _argvec[2+5] = (unsigned long)arg5;                         \
+      _argvec[2+6] = (unsigned long)arg6;                         \
+      _argvec[2+7] = (unsigned long)arg7;                         \
+      _argvec[2+8] = (unsigned long)arg8;                         \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
+         "std  2,-16(11)\n\t" /* save tocptr */                   \
+         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
+         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
+         "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
+         "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
+         "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
+         "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
+         "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
+         "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
+         "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
+         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "ld  2,-16(11)\n\t" /* restore tocptr */                 \
+         VG_CONTRACT_FRAME_BY(512)                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
+                                 arg7,arg8,arg9)                  \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+9];                        \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      _argvec[2+2] = (unsigned long)arg2;                         \
+      _argvec[2+3] = (unsigned long)arg3;                         \
+      _argvec[2+4] = (unsigned long)arg4;                         \
+      _argvec[2+5] = (unsigned long)arg5;                         \
+      _argvec[2+6] = (unsigned long)arg6;                         \
+      _argvec[2+7] = (unsigned long)arg7;                         \
+      _argvec[2+8] = (unsigned long)arg8;                         \
+      _argvec[2+9] = (unsigned long)arg9;                         \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
+         "std  2,-16(11)\n\t" /* save tocptr */                   \
+         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
+         VG_EXPAND_FRAME_BY_trashes_r3(128)                       \
+         /* arg9 */                                               \
+         "ld  3,72(11)\n\t"                                       \
+         "std 3,112(1)\n\t"                                       \
+         /* args1-8 */                                            \
+         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
+         "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
+         "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
+         "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
+         "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
+         "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
+         "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
+         "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
+         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "ld  2,-16(11)\n\t" /* restore tocptr */                 \
+         VG_CONTRACT_FRAME_BY(128)                                \
+         VG_CONTRACT_FRAME_BY(512)                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
+                                  arg7,arg8,arg9,arg10)           \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+10];                       \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      _argvec[2+2] = (unsigned long)arg2;                         \
+      _argvec[2+3] = (unsigned long)arg3;                         \
+      _argvec[2+4] = (unsigned long)arg4;                         \
+      _argvec[2+5] = (unsigned long)arg5;                         \
+      _argvec[2+6] = (unsigned long)arg6;                         \
+      _argvec[2+7] = (unsigned long)arg7;                         \
+      _argvec[2+8] = (unsigned long)arg8;                         \
+      _argvec[2+9] = (unsigned long)arg9;                         \
+      _argvec[2+10] = (unsigned long)arg10;                       \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
+         "std  2,-16(11)\n\t" /* save tocptr */                   \
+         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
+         VG_EXPAND_FRAME_BY_trashes_r3(128)                       \
+         /* arg10 */                                              \
+         "ld  3,80(11)\n\t"                                       \
+         "std 3,120(1)\n\t"                                       \
+         /* arg9 */                                               \
+         "ld  3,72(11)\n\t"                                       \
+         "std 3,112(1)\n\t"                                       \
+         /* args1-8 */                                            \
+         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
+         "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
+         "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
+         "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
+         "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
+         "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
+         "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
+         "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
+         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "ld  2,-16(11)\n\t" /* restore tocptr */                 \
+         VG_CONTRACT_FRAME_BY(128)                                \
+         VG_CONTRACT_FRAME_BY(512)                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
+                                  arg7,arg8,arg9,arg10,arg11)     \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+11];                       \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      _argvec[2+2] = (unsigned long)arg2;                         \
+      _argvec[2+3] = (unsigned long)arg3;                         \
+      _argvec[2+4] = (unsigned long)arg4;                         \
+      _argvec[2+5] = (unsigned long)arg5;                         \
+      _argvec[2+6] = (unsigned long)arg6;                         \
+      _argvec[2+7] = (unsigned long)arg7;                         \
+      _argvec[2+8] = (unsigned long)arg8;                         \
+      _argvec[2+9] = (unsigned long)arg9;                         \
+      _argvec[2+10] = (unsigned long)arg10;                       \
+      _argvec[2+11] = (unsigned long)arg11;                       \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
+         "std  2,-16(11)\n\t" /* save tocptr */                   \
+         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
+         VG_EXPAND_FRAME_BY_trashes_r3(144)                       \
+         /* arg11 */                                              \
+         "ld  3,88(11)\n\t"                                       \
+         "std 3,128(1)\n\t"                                       \
+         /* arg10 */                                              \
+         "ld  3,80(11)\n\t"                                       \
+         "std 3,120(1)\n\t"                                       \
+         /* arg9 */                                               \
+         "ld  3,72(11)\n\t"                                       \
+         "std 3,112(1)\n\t"                                       \
+         /* args1-8 */                                            \
+         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
+         "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
+         "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
+         "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
+         "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
+         "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
+         "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
+         "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
+         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "ld  2,-16(11)\n\t" /* restore tocptr */                 \
+         VG_CONTRACT_FRAME_BY(144)                                \
+         VG_CONTRACT_FRAME_BY(512)                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
+                                arg7,arg8,arg9,arg10,arg11,arg12) \
+   do {                                                           \
+      volatile OrigFn        _orig = (orig);                      \
+      volatile unsigned long _argvec[3+12];                       \
+      volatile unsigned long _res;                                \
+      /* _argvec[0] holds current r2 across the call */           \
+      _argvec[1]   = (unsigned long)_orig.r2;                     \
+      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
+      _argvec[2+1] = (unsigned long)arg1;                         \
+      _argvec[2+2] = (unsigned long)arg2;                         \
+      _argvec[2+3] = (unsigned long)arg3;                         \
+      _argvec[2+4] = (unsigned long)arg4;                         \
+      _argvec[2+5] = (unsigned long)arg5;                         \
+      _argvec[2+6] = (unsigned long)arg6;                         \
+      _argvec[2+7] = (unsigned long)arg7;                         \
+      _argvec[2+8] = (unsigned long)arg8;                         \
+      _argvec[2+9] = (unsigned long)arg9;                         \
+      _argvec[2+10] = (unsigned long)arg10;                       \
+      _argvec[2+11] = (unsigned long)arg11;                       \
+      _argvec[2+12] = (unsigned long)arg12;                       \
+      __asm__ volatile(                                           \
+         "mr 11,%1\n\t"                                           \
+         VG_EXPAND_FRAME_BY_trashes_r3(512)                       \
+         "std  2,-16(11)\n\t" /* save tocptr */                   \
+         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
+         VG_EXPAND_FRAME_BY_trashes_r3(144)                       \
+         /* arg12 */                                              \
+         "ld  3,96(11)\n\t"                                       \
+         "std 3,136(1)\n\t"                                       \
+         /* arg11 */                                              \
+         "ld  3,88(11)\n\t"                                       \
+         "std 3,128(1)\n\t"                                       \
+         /* arg10 */                                              \
+         "ld  3,80(11)\n\t"                                       \
+         "std 3,120(1)\n\t"                                       \
+         /* arg9 */                                               \
+         "ld  3,72(11)\n\t"                                       \
+         "std 3,112(1)\n\t"                                       \
+         /* args1-8 */                                            \
+         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
+         "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
+         "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
+         "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
+         "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
+         "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
+         "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
+         "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
+         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
+         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
+         "mr 11,%1\n\t"                                           \
+         "mr %0,3\n\t"                                            \
+         "ld  2,-16(11)\n\t" /* restore tocptr */                 \
+         VG_CONTRACT_FRAME_BY(144)                                \
+         VG_CONTRACT_FRAME_BY(512)                                \
+         : /*out*/   "=r" (_res)                                  \
+         : /*in*/    "r" (&_argvec[2])                            \
+         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS          \
+      );                                                          \
+      lval = (__typeof__(lval)) _res;                             \
+   } while (0)
+
+#endif /* PLAT_ppc64_aix5 */
+
+
+/* ------------------------------------------------------------------ */
+/* ARCHITECTURE INDEPENDENT MACROS for CLIENT REQUESTS.               */
+/*                                                                    */
+/* ------------------------------------------------------------------ */
+
+/* Some request codes.  There are many more of these, but most are not
+   exposed to end-user view.  These are the public ones, all of the
+   form 0x1000 + small_number.
+
+   Core ones are in the range 0x00000000--0x0000ffff.  The non-public
+   ones start at 0x2000.
+*/
+
+/* These macros are used by tools -- they must be public, but don't
+   embed them into other programs. */
+#define VG_USERREQ_TOOL_BASE(a,b) \
+   ((unsigned int)(((a)&0xff) << 24 | ((b)&0xff) << 16))
+#define VG_IS_TOOL_USERREQ(a, b, v) \
+   (VG_USERREQ_TOOL_BASE(a,b) == ((v) & 0xffff0000))
+
+/* !! ABIWARNING !! ABIWARNING !! ABIWARNING !! ABIWARNING !! 
+   This enum comprises an ABI exported by Valgrind to programs
+   which use client requests.  DO NOT CHANGE THE ORDER OF THESE
+   ENTRIES, NOR DELETE ANY -- add new ones at the end. */
+typedef
+   enum { VG_USERREQ__RUNNING_ON_VALGRIND  = 0x1001,
+          VG_USERREQ__DISCARD_TRANSLATIONS = 0x1002,
+
+          /* These allow any function to be called from the simulated
+             CPU but run on the real CPU.  Nb: the first arg passed to
+             the function is always the ThreadId of the running
+             thread!  So CLIENT_CALL0 actually requires a 1 arg
+             function, etc. */
+          VG_USERREQ__CLIENT_CALL0 = 0x1101,
+          VG_USERREQ__CLIENT_CALL1 = 0x1102,
+          VG_USERREQ__CLIENT_CALL2 = 0x1103,
+          VG_USERREQ__CLIENT_CALL3 = 0x1104,
+
+          /* Can be useful in regression testing suites -- eg. can
+             send Valgrind's output to /dev/null and still count
+             errors. */
+          VG_USERREQ__COUNT_ERRORS = 0x1201,
+
+          /* These are useful and can be interpreted by any tool that
+             tracks malloc() et al, by using vg_replace_malloc.c. */
+          VG_USERREQ__MALLOCLIKE_BLOCK = 0x1301,
+          VG_USERREQ__FREELIKE_BLOCK   = 0x1302,
+          /* Memory pool support. */
+          VG_USERREQ__CREATE_MEMPOOL   = 0x1303,
+          VG_USERREQ__DESTROY_MEMPOOL  = 0x1304,
+          VG_USERREQ__MEMPOOL_ALLOC    = 0x1305,
+          VG_USERREQ__MEMPOOL_FREE     = 0x1306,
+          VG_USERREQ__MEMPOOL_TRIM     = 0x1307,
+          VG_USERREQ__MOVE_MEMPOOL     = 0x1308,
+          VG_USERREQ__MEMPOOL_CHANGE   = 0x1309,
+          VG_USERREQ__MEMPOOL_EXISTS   = 0x130a,
+
+          /* Allow printfs to valgrind log. */
+          VG_USERREQ__PRINTF           = 0x1401,
+          VG_USERREQ__PRINTF_BACKTRACE = 0x1402,
+
+          /* Stack support. */
+          VG_USERREQ__STACK_REGISTER   = 0x1501,
+          VG_USERREQ__STACK_DEREGISTER = 0x1502,
+          VG_USERREQ__STACK_CHANGE     = 0x1503
+   } Vg_ClientRequest;
+
+#if !defined(__GNUC__)
+#  define __extension__ /* */
+#endif
+
+/* Returns the number of Valgrinds this code is running under.  That
+   is, 0 if running natively, 1 if running under Valgrind, 2 if
+   running under Valgrind which is running under another Valgrind,
+   etc. */
+#define RUNNING_ON_VALGRIND  __extension__                        \
+   ({unsigned int _qzz_res;                                       \
+    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0 /* if not */,          \
+                               VG_USERREQ__RUNNING_ON_VALGRIND,   \
+                               0, 0, 0, 0, 0);                    \
+    _qzz_res;                                                     \
+   })
+
+
+/* Discard translation of code in the range [_qzz_addr .. _qzz_addr +
+   _qzz_len - 1].  Useful if you are debugging a JITter or some such,
+   since it provides a way to make sure valgrind will retranslate the
+   invalidated area.  Returns no value. */
+#define VALGRIND_DISCARD_TRANSLATIONS(_qzz_addr,_qzz_len)         \
+   {unsigned int _qzz_res;                                        \
+    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
+                               VG_USERREQ__DISCARD_TRANSLATIONS,  \
+                               _qzz_addr, _qzz_len, 0, 0, 0);     \
+   }
+
+
+/* These requests are for getting Valgrind itself to print something.
+   Possibly with a backtrace.  This is a really ugly hack. */
+
+#if defined(NVALGRIND)
+
+#  define VALGRIND_PRINTF(...)
+#  define VALGRIND_PRINTF_BACKTRACE(...)
+
+#else /* NVALGRIND */
+
+/* Modern GCC will optimize the static routine out if unused,
+   and unused attribute will shut down warnings about it.  */
+static int VALGRIND_PRINTF(const char *format, ...)
+   __attribute__((format(__printf__, 1, 2), __unused__));
+static int
+VALGRIND_PRINTF(const char *format, ...)
+{
+   unsigned long _qzz_res;
+   va_list vargs;
+   va_start(vargs, format);
+   VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, VG_USERREQ__PRINTF,
+                              (unsigned long)format, (unsigned long)vargs, 
+                              0, 0, 0);
+   va_end(vargs);
+   return (int)_qzz_res;
+}
+
+static int VALGRIND_PRINTF_BACKTRACE(const char *format, ...)
+   __attribute__((format(__printf__, 1, 2), __unused__));
+static int
+VALGRIND_PRINTF_BACKTRACE(const char *format, ...)
+{
+   unsigned long _qzz_res;
+   va_list vargs;
+   va_start(vargs, format);
+   VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0, VG_USERREQ__PRINTF_BACKTRACE,
+                              (unsigned long)format, (unsigned long)vargs, 
+                              0, 0, 0);
+   va_end(vargs);
+   return (int)_qzz_res;
+}
+
+#endif /* NVALGRIND */
+
+
+/* These requests allow control to move from the simulated CPU to the
+   real CPU, calling an arbitary function.
+   
+   Note that the current ThreadId is inserted as the first argument.
+   So this call:
+
+     VALGRIND_NON_SIMD_CALL2(f, arg1, arg2)
+
+   requires f to have this signature:
+
+     Word f(Word tid, Word arg1, Word arg2)
+
+   where "Word" is a word-sized type.
+
+   Note that these client requests are not entirely reliable.  For example,
+   if you call a function with them that subsequently calls printf(),
+   there's a high chance Valgrind will crash.  Generally, your prospects of
+   these working are made higher if the called function does not refer to
+   any global variables, and does not refer to any libc or other functions
+   (printf et al).  Any kind of entanglement with libc or dynamic linking is
+   likely to have a bad outcome, for tricky reasons which we've grappled
+   with a lot in the past.
+*/
+#define VALGRIND_NON_SIMD_CALL0(_qyy_fn)                          \
+   __extension__                                                  \
+   ({unsigned long _qyy_res;                                      \
+    VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */,  \
+                               VG_USERREQ__CLIENT_CALL0,          \
+                               _qyy_fn,                           \
+                               0, 0, 0, 0);                       \
+    _qyy_res;                                                     \
+   })
+
+#define VALGRIND_NON_SIMD_CALL1(_qyy_fn, _qyy_arg1)               \
+   __extension__                                                  \
+   ({unsigned long _qyy_res;                                      \
+    VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */,  \
+                               VG_USERREQ__CLIENT_CALL1,          \
+                               _qyy_fn,                           \
+                               _qyy_arg1, 0, 0, 0);               \
+    _qyy_res;                                                     \
+   })
+
+#define VALGRIND_NON_SIMD_CALL2(_qyy_fn, _qyy_arg1, _qyy_arg2)    \
+   __extension__                                                  \
+   ({unsigned long _qyy_res;                                      \
+    VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */,  \
+                               VG_USERREQ__CLIENT_CALL2,          \
+                               _qyy_fn,                           \
+                               _qyy_arg1, _qyy_arg2, 0, 0);       \
+    _qyy_res;                                                     \
+   })
+
+#define VALGRIND_NON_SIMD_CALL3(_qyy_fn, _qyy_arg1, _qyy_arg2, _qyy_arg3) \
+   __extension__                                                  \
+   ({unsigned long _qyy_res;                                      \
+    VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */,  \
+                               VG_USERREQ__CLIENT_CALL3,          \
+                               _qyy_fn,                           \
+                               _qyy_arg1, _qyy_arg2,              \
+                               _qyy_arg3, 0);                     \
+    _qyy_res;                                                     \
+   })
+
+
+/* Counts the number of errors that have been recorded by a tool.  Nb:
+   the tool must record the errors with VG_(maybe_record_error)() or
+   VG_(unique_error)() for them to be counted. */
+#define VALGRIND_COUNT_ERRORS                                     \
+   __extension__                                                  \
+   ({unsigned int _qyy_res;                                       \
+    VALGRIND_DO_CLIENT_REQUEST(_qyy_res, 0 /* default return */,  \
+                               VG_USERREQ__COUNT_ERRORS,          \
+                               0, 0, 0, 0, 0);                    \
+    _qyy_res;                                                     \
+   })
+
+/* Mark a block of memory as having been allocated by a malloc()-like
+   function.  `addr' is the start of the usable block (ie. after any
+   redzone) `rzB' is redzone size if the allocator can apply redzones;
+   use '0' if not.  Adding redzones makes it more likely Valgrind will spot
+   block overruns.  `is_zeroed' indicates if the memory is zeroed, as it is
+   for calloc().  Put it immediately after the point where a block is
+   allocated. 
+   
+   If you're using Memcheck: If you're allocating memory via superblocks,
+   and then handing out small chunks of each superblock, if you don't have
+   redzones on your small blocks, it's worth marking the superblock with
+   VALGRIND_MAKE_MEM_NOACCESS when it's created, so that block overruns are
+   detected.  But if you can put redzones on, it's probably better to not do
+   this, so that messages for small overruns are described in terms of the
+   small block rather than the superblock (but if you have a big overrun
+   that skips over a redzone, you could miss an error this way).  See
+   memcheck/tests/custom_alloc.c for an example.
+
+   WARNING: if your allocator uses malloc() or 'new' to allocate
+   superblocks, rather than mmap() or brk(), this will not work properly --
+   you'll likely get assertion failures during leak detection.  This is
+   because Valgrind doesn't like seeing overlapping heap blocks.  Sorry.
+
+   Nb: block must be freed via a free()-like function specified
+   with VALGRIND_FREELIKE_BLOCK or mismatch errors will occur. */
+#define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed)    \
+   {unsigned int _qzz_res;                                        \
+    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
+                               VG_USERREQ__MALLOCLIKE_BLOCK,      \
+                               addr, sizeB, rzB, is_zeroed, 0);   \
+   }
+
+/* Mark a block of memory as having been freed by a free()-like function.
+   `rzB' is redzone size;  it must match that given to
+   VALGRIND_MALLOCLIKE_BLOCK.  Memory not freed will be detected by the leak
+   checker.  Put it immediately after the point where the block is freed. */
+#define VALGRIND_FREELIKE_BLOCK(addr, rzB)                        \
+   {unsigned int _qzz_res;                                        \
+    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
+                               VG_USERREQ__FREELIKE_BLOCK,        \
+                               addr, rzB, 0, 0, 0);               \
+   }
+
+/* Create a memory pool. */
+#define VALGRIND_CREATE_MEMPOOL(pool, rzB, is_zeroed)             \
+   {unsigned int _qzz_res;                                        \
+    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
+                               VG_USERREQ__CREATE_MEMPOOL,        \
+                               pool, rzB, is_zeroed, 0, 0);       \
+   }
+
+/* Destroy a memory pool. */
+#define VALGRIND_DESTROY_MEMPOOL(pool)                            \
+   {unsigned int _qzz_res;                                        \
+    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
+                               VG_USERREQ__DESTROY_MEMPOOL,       \
+                               pool, 0, 0, 0, 0);                 \
+   }
+
+/* Associate a piece of memory with a memory pool. */
+#define VALGRIND_MEMPOOL_ALLOC(pool, addr, size)                  \
+   {unsigned int _qzz_res;                                        \
+    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
+                               VG_USERREQ__MEMPOOL_ALLOC,         \
+                               pool, addr, size, 0, 0);           \
+   }
+
+/* Disassociate a piece of memory from a memory pool. */
+#define VALGRIND_MEMPOOL_FREE(pool, addr)                         \
+   {unsigned int _qzz_res;                                        \
+    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
+                               VG_USERREQ__MEMPOOL_FREE,          \
+                               pool, addr, 0, 0, 0);              \
+   }
+
+/* Disassociate any pieces outside a particular range. */
+#define VALGRIND_MEMPOOL_TRIM(pool, addr, size)                   \
+   {unsigned int _qzz_res;                                        \
+    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
+                               VG_USERREQ__MEMPOOL_TRIM,          \
+                               pool, addr, size, 0, 0);           \
+   }
+
+/* Resize and/or move a piece associated with a memory pool. */
+#define VALGRIND_MOVE_MEMPOOL(poolA, poolB)                       \
+   {unsigned int _qzz_res;                                        \
+    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
+                               VG_USERREQ__MOVE_MEMPOOL,          \
+                               poolA, poolB, 0, 0, 0);            \
+   }
+
+/* Resize and/or move a piece associated with a memory pool. */
+#define VALGRIND_MEMPOOL_CHANGE(pool, addrA, addrB, size)         \
+   {unsigned int _qzz_res;                                        \
+    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
+                               VG_USERREQ__MEMPOOL_CHANGE,        \
+                               pool, addrA, addrB, size, 0);      \
+   }
+
+/* Return 1 if a mempool exists, else 0. */
+#define VALGRIND_MEMPOOL_EXISTS(pool)                             \
+   __extension__                                                  \
+   ({unsigned int _qzz_res;                                       \
+    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
+                               VG_USERREQ__MEMPOOL_EXISTS,        \
+                               pool, 0, 0, 0, 0);                 \
+    _qzz_res;                                                     \
+   })
+
+/* Mark a piece of memory as being a stack. Returns a stack id. */
+#define VALGRIND_STACK_REGISTER(start, end)                       \
+   __extension__                                                  \
+   ({unsigned int _qzz_res;                                       \
+    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
+                               VG_USERREQ__STACK_REGISTER,        \
+                               start, end, 0, 0, 0);              \
+    _qzz_res;                                                     \
+   })
+
+/* Unmark the piece of memory associated with a stack id as being a
+   stack. */
+#define VALGRIND_STACK_DEREGISTER(id)                             \
+   {unsigned int _qzz_res;                                        \
+    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
+                               VG_USERREQ__STACK_DEREGISTER,      \
+                               id, 0, 0, 0, 0);                   \
+   }
+
+/* Change the start and end address of the stack id. */
+#define VALGRIND_STACK_CHANGE(id, start, end)                     \
+   {unsigned int _qzz_res;                                        \
+    VALGRIND_DO_CLIENT_REQUEST(_qzz_res, 0,                       \
+                               VG_USERREQ__STACK_CHANGE,          \
+                               id, start, end, 0, 0);             \
+   }
+
+
+#undef PLAT_x86_linux
+#undef PLAT_amd64_linux
+#undef PLAT_ppc32_linux
+#undef PLAT_ppc64_linux
+#undef PLAT_ppc32_aix5
+#undef PLAT_ppc64_aix5
+
+#endif   /* __VALGRIND_H */