[x86] Fix some asm problems

A bag of assembler fixes, to be either compatible with GAS
(e.g. order of 'test' operands), accept more instructions,
count correct foo{bwlq} variants on x86_64, fix modrm/sib bytes
on x86_64 to not use %rip relative addressing mode, to not use
invalid insns in tests/asmtest.S for x86_64.

Result is that now output of GAS and of tcc on tests/asmtest.S
is mostly the same.

1 files changed, 71 insertions, 16 deletions

diff --git a/i386-asm.c b/i386-asm.c
index 772b74a..435dd15 100644
--- a/i386-asm.c
+++ b/i386-asm.c
@@ -115,11 +115,13 @@ enum {
 #define OP_ADDR   (1 << OPT_ADDR)
 #define OP_INDIR  (1 << OPT_INDIR)
 #ifdef TCC_TARGET_X86_64
-# define OP_REG64  (1 << OPT_REG64)
-# define OP_IM64   (1 << OPT_IM64)
+# define OP_REG64 (1 << OPT_REG64)
+# define OP_IM64  (1 << OPT_IM64)
+# define OP_EA32  (OP_EA << 1)
 #else
 # define OP_REG64 0
 # define OP_IM64  0
+# define OP_EA32  0
 #endif
 
 #define OP_EA     0x40000000
@@ -268,15 +270,17 @@ static inline int get_reg_shift(TCCState *s1)
     return shift;
 }
 
-static int asm_parse_reg(void)
+static int asm_parse_reg(int *type)
 {
     int reg = 0;
+    *type = 0;
     if (tok != '%')
         goto error_32;
     next();
     if (tok >= TOK_ASM_eax && tok <= TOK_ASM_edi) {
         reg = tok - TOK_ASM_eax;
 #ifdef TCC_TARGET_X86_64
+	*type = OP_EA32;
     } else if (tok >= TOK_ASM_rax && tok <= TOK_ASM_rdi) {
         reg = tok - TOK_ASM_rax;
 #endif
@@ -393,20 +397,23 @@ static void parse_operand(TCCState *s1, Operand *op)
             }
         }
         if (tok == '(') {
+	    int type = 0;
             next();
             if (tok != ',') {
-                op->reg = asm_parse_reg();
+                op->reg = asm_parse_reg(&type);
             }
             if (tok == ',') {
                 next();
                 if (tok != ',') {
-                    op->reg2 = asm_parse_reg();
+                    op->reg2 = asm_parse_reg(&type);
                 }
                 if (tok == ',') {
                     next();
                     op->shift = get_reg_shift(s1);
                 }
             }
+	    if (type & OP_EA32)
+	        op->type |= OP_EA32;
             skip(')');
         }
         if (op->reg == -1 && op->reg2 == -1)
@@ -493,10 +500,15 @@ static inline void asm_modrm(int reg, Operand *op)
             gen_expr16(&op->e);
         } else if (tcc_state->seg_size == 32)
 #endif
-        {
+	{
+#ifdef TCC_TARGET_X86_64
+	    g(0x04 + (reg << 3));
+	    g(0x25);
+#else
             g(0x05 + (reg << 3));
-            gen_expr32(&op->e);
-        }
+#endif
+	    gen_expr32(&op->e);
+	}
     } else {
         sib_reg1 = op->reg;
         /* fist compute displacement encoding */
@@ -583,6 +595,8 @@ ST_FUNC void asm_opcode(TCCState *s1, int opcode)
     int nb_ops, s;
     Operand ops[MAX_OPERANDS], *pop;
     int op_type[3]; /* decoded op type */
+    int alltypes;   /* OR of all operand types */
+    int autosize;
 #ifdef I386_ASM_16
     static int a32 = 0, o32 = 0, addr32 = 0, data32 = 0;
 #endif
@@ -596,6 +610,7 @@ ST_FUNC void asm_opcode(TCCState *s1, int opcode)
     pop = ops;
     nb_ops = 0;
     seg_prefix = 0;
+    alltypes = 0;
     for(;;) {
         if (tok == ';' || tok == TOK_LINEFEED)
             break;
@@ -645,6 +660,13 @@ ST_FUNC void asm_opcode(TCCState *s1, int opcode)
             if (!(opcode >= pa->sym && opcode < pa->sym + NB_TEST_OPCODES))
                 continue;
         } else if (pa->instr_type & OPC_B) {
+#ifdef TCC_TARGET_X86_64
+	    /* Some instructions don't have the full size but only
+	       bwl form.  insb e.g. */
+	    if ((pa->instr_type & OPC_WLQ) != OPC_WLQ
+		&& !(opcode >= pa->sym && opcode < pa->sym + NBWLX-1))
+	        continue;
+#endif
             if (!(opcode >= pa->sym && opcode < pa->sym + NBWLX))
                 continue;
             s = opcode - pa->sym;
@@ -659,6 +681,7 @@ ST_FUNC void asm_opcode(TCCState *s1, int opcode)
         if (pa->nb_ops != nb_ops)
             continue;
         /* now decode and check each operand */
+	alltypes = 0;
         for(i = 0; i < nb_ops; i++) {
             int op1, op2;
             op1 = pa->op_type[i];
@@ -687,9 +710,10 @@ ST_FUNC void asm_opcode(TCCState *s1, int opcode)
             }
             if (op1 & OPT_EA)
                 v |= OP_EA;
-            op_type[i] = v;
+	    op_type[i] = v;
             if ((ops[i].type & v) == 0)
                 goto next;
+	    alltypes |= ops[i].type;
         }
         /* all is matching ! */
         break;
@@ -725,12 +749,19 @@ ST_FUNC void asm_opcode(TCCState *s1, int opcode)
         }
     }
     /* if the size is unknown, then evaluate it (OPC_B or OPC_WL case) */
-    if (s == NBWLX-1) {
-        for(i = 0; s == NBWLX-1 && i < nb_ops; i++) {
+    autosize = NBWLX-1;
+#ifdef TCC_TARGET_X86_64
+    /* XXX the autosize should rather be zero, to not have to adjust this
+       all the time.  */
+    if ((pa->instr_type & OPC_WLQ) != OPC_WLQ)
+        autosize = NBWLX-2;
+#endif
+    if (s == autosize) {
+        for(i = 0; s == autosize && i < nb_ops; i++) {
             if ((ops[i].type & OP_REG) && !(op_type[i] & (OP_CL | OP_DX)))
                 s = reg_to_size[ops[i].type & OP_REG];
         }
-        if (s == NBWLX-1) {
+        if (s == autosize) {
             if ((opcode == TOK_ASM_push || opcode == TOK_ASM_pop) &&
                 (ops[0].type & (OP_SEG | OP_IM8S | OP_IM32 | OP_IM64)))
                 s = 2;
@@ -770,14 +801,38 @@ ST_FUNC void asm_opcode(TCCState *s1, int opcode)
 
     addr32 = data32 = 0;
 #else
+#ifdef TCC_TARGET_X86_64
+    /* Generate addr32 prefix if needed */
+    for(i = 0; i < nb_ops; i++) {
+        if (ops[i].type & OP_EA32) {
+	    g(0x67);
+	    break;
+        }
+    }
+#endif
     /* generate data16 prefix if needed */
     if (s == 1 || (pa->instr_type & OPC_D16))
         g(0x66);
 #ifdef TCC_TARGET_X86_64
-    else if (s == 3) {
+    if (s == 3 || (alltypes & OP_REG64)) {
         /* generate REX prefix */
-        if ((opcode != TOK_ASM_push && opcode != TOK_ASM_pop)
-            || !(ops[0].type & OP_REG64))
+	int default64 = 0;
+	for(i = 0; i < nb_ops; i++) {
+	    if (op_type[i] == OP_REG64) {
+		/* If only 64bit regs are accepted in one operand
+		   this is a default64 instruction without need for
+		   REX prefixes.  */
+		default64 = 1;
+		break;
+	    }
+	}
+	/* XXX find better encoding for the default64 instructions.  */
+        if (((opcode != TOK_ASM_push && opcode != TOK_ASM_pop
+	      && opcode != TOK_ASM_pushw && opcode != TOK_ASM_pushl
+	      && opcode != TOK_ASM_pushq && opcode != TOK_ASM_popw
+	      && opcode != TOK_ASM_popl && opcode != TOK_ASM_popq
+	      && opcode != TOK_ASM_call && opcode != TOK_ASM_jmp))
+	    && !default64)
             g(0x48);
     }
 #endif
@@ -838,7 +893,7 @@ ST_FUNC void asm_opcode(TCCState *s1, int opcode)
             goto no_short_jump;
         if (sym->r != cur_text_section->sh_num)
             goto no_short_jump;
-        jmp_disp = ops[0].e.v + sym->jnext - ind - 2;
+        jmp_disp = ops[0].e.v + sym->jnext - ind - 2 - (v >= 0xff);
         if (jmp_disp == (int8_t)jmp_disp) {
             /* OK to generate jump */
             is_short_jmp = 1;