fix a subtle x86-64 calling bug

I ran into an issue playing with tinycc, and tracked it down to a rather
weird assumption in the function calling code. This breaks only when
varargs and float/double arguments are combined, I think, and only when
calling GCC-generated (or non-TinyCC, at least) code. The problem is we
sometimes generate code like this:

804a468: 4c 89 d9 mov %r11,%rcx
804a46b: b8 01 00 00 00 mov $0x1,%eax
804a470: 48 8b 45 c0 mov -0x40(%rbp),%rax
804a474: 4c 8b 18 mov (%rax),%r11
804a477: 41 ff d3 callq *%r11

for a function call. Note how $eax is first set to the correct value,
then clobbered when we try to load the function pointer into R11. With
the patch, the code generated is:

804a468: 4c 89 d9 mov %r11,%rcx
804a46b: b8 01 00 00 00 mov $0x1,%eax
804a470: 4c 8b 5d c0 mov -0x40(%rbp),%r11
804a474: 4d 8b 1b mov (%r11),%r11
804a477: 41 ff d3 callq *%r11

which is correct.

This becomes an issue when get_reg(RC_INT) is modified not always to
return %rax after a save_regs(0), because then another register (%ecx,
say) is clobbered, and the function passed an invalid argument.

A rather convoluted test case that generates the above code is
included. Please note that the test will not cause a failure because
TinyCC code ignores the %rax argument, but it will cause incorrect
behavior when combined with GCC code, which might wrongly fail to save
XMM registers and cause data corruption.

1 files changed, 37 insertions, 0 deletions

diff --git a/tests/abitest.c b/tests/abitest.c
index d3e151f..bf67f92 100644
--- a/tests/abitest.c
+++ b/tests/abitest.c
@@ -316,6 +316,42 @@ static int many_struct_test_2(void) {
 }
 
 /*
+ * Win64 calling convention test.
+ */
+
+typedef struct many_struct_test_3_type_s {int a, b;} many_struct_test_3_type;
+typedef many_struct_test_3_type (*many_struct_test_3_function_type) (many_struct_test_3_type,many_struct_test_3_type,many_struct_test_3_type,many_struct_test_3_type,many_struct_test_3_type,many_struct_test_3_type, ...);
+typedef struct many_struct_test_3_struct_type { many_struct_test_3_function_type f; many_struct_test_3_function_type *f2; } many_struct_test_3_struct_type;
+
+static void many_struct_test_3_dummy(double d, ...)
+{
+  volatile double x = d;
+}
+
+static int many_struct_test_3_callback(void *ptr) {
+  many_struct_test_3_struct_type s = { ptr, };
+  many_struct_test_3_struct_type *s2 = &s;
+  s2->f2 = &s2->f;
+  many_struct_test_3_dummy(1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, &s2);
+  many_struct_test_3_function_type f = *(s2->f2);
+  many_struct_test_3_type v = {1,2};
+  many_struct_test_3_type r = (*((s2->f2=&f)+0))(v,v,v,v,v,v,1.0);
+  return ((r.a == 6) && (r.b == 12))?0:-1;
+}
+
+static int many_struct_test_3(void) {
+  const char *src =
+  "typedef struct many_struct_test_3_type_s {int a, b;} many_struct_test_3_type;\n"
+  "many_struct_test_3_type f(many_struct_test_3_type x1, many_struct_test_3_type x2, many_struct_test_3_type x3, many_struct_test_3_type x4, many_struct_test_3_type x5, many_struct_test_3_type x6, ...) {\n"
+  "  many_struct_test_3_type y;\n"
+  "  y.a = x1.a + x2.a + x3.a + x4.a + x5.a + x6.a;\n"
+  "  y.b = x1.b + x2.b + x3.b + x4.b + x5.b + x6.b;\n"
+  "  return y;\n"
+  "}\n";
+  return run_callback(src, many_struct_test_3_callback);
+}
+
+/*
  * stdarg_test: Test variable argument list ABI
  */
 
@@ -448,6 +484,7 @@ int main(int argc, char **argv) {
   RUN_TEST(two_member_union_test);
   RUN_TEST(many_struct_test);
   RUN_TEST(many_struct_test_2);
+  RUN_TEST(many_struct_test_3);
   RUN_TEST(stdarg_test);
   RUN_TEST(stdarg_struct_test);
   RUN_TEST(arg_align_test);