6911 - comparing floats

It turns out floating-point operations set different flags than most instructions.
We have to branch on them using unsigned jumps.

https://stackoverflow.com/questions/7057501/x86-assembler-floating-point-compare/7057771#7057771

5 files changed, 63 insertions, 23 deletions

diff --git a/014indirect_addressing.cc b/014indirect_addressing.cc
index fa679d6f..04b65215 100644
--- a/014indirect_addressing.cc
+++ b/014indirect_addressing.cc
@@ -538,7 +538,7 @@ void test_compare_mem_at_r32_with_r32_equal() {
 put_new(Name, "3b", "compare: set SF if r32 < rm32 (cmp)");
 
 :(code)
-void test_compare_r32_with_mem_at_r32_greater() {
+void test_compare_r32_with_mem_at_rm32_greater() {
   Reg[EAX].i = 0x2000;
   Reg[EBX].i = 0x0a0b0c0d;
   run(
@@ -576,7 +576,7 @@ case 0x3b: {  // set SF if r32 < r/m32
 }
 
 :(code)
-void test_compare_r32_with_mem_at_r32_lesser_unsigned_and_signed() {
+void test_compare_r32_with_mem_at_rm32_lesser_unsigned_and_signed() {
   Reg[EAX].i = 0x2000;
   Reg[EBX].i = 0x0a0b0c07;
   run(
@@ -595,7 +595,7 @@ void test_compare_r32_with_mem_at_r32_lesser_unsigned_and_signed() {
   );
 }
 
-void test_compare_r32_with_mem_at_r32_lesser_unsigned_and_signed_due_to_overflow() {
+void test_compare_r32_with_mem_at_rm32_lesser_unsigned_and_signed_due_to_overflow() {
   Reg[EAX].i = 0x2000;
   Reg[EBX].i = 0x7fffffff;  // largest positive signed integer
   run(
@@ -614,7 +614,7 @@ void test_compare_r32_with_mem_at_r32_lesser_unsigned_and_signed_due_to_overflow
   );
 }
 
-void test_compare_r32_with_mem_at_r32_lesser_signed() {
+void test_compare_r32_with_mem_at_rm32_lesser_signed() {
   Reg[EAX].i = 0x2000;
   Reg[EBX].i = 0xffffffff;  // -1
   run(
@@ -633,7 +633,7 @@ void test_compare_r32_with_mem_at_r32_lesser_signed() {
   );
 }
 
-void test_compare_r32_with_mem_at_r32_lesser_unsigned() {
+void test_compare_r32_with_mem_at_rm32_lesser_unsigned() {
   Reg[EAX].i = 0x2000;
   Reg[EBX].i = 0x00000001;  // 1
   run(
@@ -652,7 +652,7 @@ void test_compare_r32_with_mem_at_r32_lesser_unsigned() {
   );
 }
 
-void test_compare_r32_with_mem_at_r32_equal() {
+void test_compare_r32_with_mem_at_rm32_equal() {
   Reg[EAX].i = 0x2000;
   Reg[EBX].i = 0x0a0b0c0d;
   run(
diff --git a/017jump_disp8.cc b/017jump_disp8.cc
index 47a94ea9..30e60a74 100644
--- a/017jump_disp8.cc
+++ b/017jump_disp8.cc
@@ -135,8 +135,8 @@ void test_jne_disp8_fail() {
 //:: jump if greater
 
 :(before "End Initialize Op Names")
-put_new(Name, "7f", "jump disp8 bytes away if greater (signed), if ZF is unset and SF == OF (jcc/jg/jnle)");
-put_new(Name, "77", "jump disp8 bytes away if greater (unsigned), if ZF is unset and CF is unset (jcc/ja/jnbe)");
+put_new(Name, "7f", "jump disp8 bytes away if greater, if ZF is unset and SF == OF (jcc/jg/jnle)");
+put_new(Name, "77", "jump disp8 bytes away if greater (addr, float), if ZF is unset and CF is unset (jcc/ja/jnbe)");
 
 :(code)
 void test_jg_disp8_success() {
@@ -199,8 +199,8 @@ void test_jg_disp8_fail() {
 //:: jump if greater or equal
 
 :(before "End Initialize Op Names")
-put_new(Name, "7d", "jump disp8 bytes away if greater or equal (signed), if SF == OF (jcc/jge/jnl)");
-put_new(Name, "73", "jump disp8 bytes away if greater or equal (unsigned), if CF is unset (jcc/jae/jnb)");
+put_new(Name, "7d", "jump disp8 bytes away if greater or equal, if SF == OF (jcc/jge/jnl)");
+put_new(Name, "73", "jump disp8 bytes away if greater or equal (addr, float), if CF is unset (jcc/jae/jnb)");
 
 :(code)
 void test_jge_disp8_success() {
@@ -261,8 +261,8 @@ void test_jge_disp8_fail() {
 //:: jump if lesser
 
 :(before "End Initialize Op Names")
-put_new(Name, "7c", "jump disp8 bytes away if lesser (signed), if SF != OF (jcc/jl/jnge)");
-put_new(Name, "72", "jump disp8 bytes away if lesser (unsigned), if CF is set (jcc/jb/jnae)");
+put_new(Name, "7c", "jump disp8 bytes away if lesser, if SF != OF (jcc/jl/jnge)");
+put_new(Name, "72", "jump disp8 bytes away if lesser (addr, float), if CF is set (jcc/jb/jnae)");
 
 :(code)
 void test_jl_disp8_success() {
@@ -325,8 +325,8 @@ void test_jl_disp8_fail() {
 //:: jump if lesser or equal
 
 :(before "End Initialize Op Names")
-put_new(Name, "7e", "jump disp8 bytes away if lesser or equal (signed), if ZF is set or SF != OF (jcc/jle/jng)");
-put_new(Name, "76", "jump disp8 bytes away if lesser or equal (unsigned), if ZF is set or CF is set (jcc/jbe/jna)");
+put_new(Name, "7e", "jump disp8 bytes away if lesser or equal, if ZF is set or SF != OF (jcc/jle/jng)");
+put_new(Name, "76", "jump disp8 bytes away if lesser or equal (addr, float), if ZF is set or CF is set (jcc/jbe/jna)");
 
 :(code)
 void test_jle_disp8_equal() {
diff --git a/018jump_disp32.cc b/018jump_disp32.cc
index 4166fb60..e77bc584 100644
--- a/018jump_disp32.cc
+++ b/018jump_disp32.cc
@@ -135,8 +135,8 @@ void test_jne_disp32_fail() {
 //:: jump if greater
 
 :(before "End Initialize Op Names")
-put_new(Name_0f, "8f", "jump disp32 bytes away if greater (signed), if ZF is unset and SF == OF (jcc/jg/jnle)");
-put_new(Name_0f, "87", "jump disp32 bytes away if greater (unsigned), if ZF is unset and CF is unset (jcc/ja/jnbe)");
+put_new(Name_0f, "8f", "jump disp32 bytes away if greater, if ZF is unset and SF == OF (jcc/jg/jnle)");
+put_new(Name_0f, "87", "jump disp32 bytes away if greater (addr, float), if ZF is unset and CF is unset (jcc/ja/jnbe)");
 
 :(code)
 void test_jg_disp32_success() {
@@ -199,8 +199,8 @@ void test_jg_disp32_fail() {
 //:: jump if greater or equal
 
 :(before "End Initialize Op Names")
-put_new(Name_0f, "8d", "jump disp32 bytes away if greater or equal (signed), if SF == OF (jcc/jge/jnl)");
-put_new(Name_0f, "83", "jump disp32 bytes away if greater or equal (unsigned), if CF is unset (jcc/jae/jnb)");
+put_new(Name_0f, "8d", "jump disp32 bytes away if greater or equal, if SF == OF (jcc/jge/jnl)");
+put_new(Name_0f, "83", "jump disp32 bytes away if greater or equal (addr, float), if CF is unset (jcc/jae/jnb)");
 
 :(code)
 void test_jge_disp32_success() {
@@ -261,8 +261,8 @@ void test_jge_disp32_fail() {
 //:: jump if lesser
 
 :(before "End Initialize Op Names")
-put_new(Name_0f, "8c", "jump disp32 bytes away if lesser (signed), if SF != OF (jcc/jl/jnge)");
-put_new(Name_0f, "82", "jump disp32 bytes away if lesser (unsigned), if CF is set (jcc/jb/jnae)");
+put_new(Name_0f, "8c", "jump disp32 bytes away if lesser, if SF != OF (jcc/jl/jnge)");
+put_new(Name_0f, "82", "jump disp32 bytes away if lesser (addr, float), if CF is set (jcc/jb/jnae)");
 
 :(code)
 void test_jl_disp32_success() {
@@ -325,8 +325,8 @@ void test_jl_disp32_fail() {
 //:: jump if lesser or equal
 
 :(before "End Initialize Op Names")
-put_new(Name_0f, "8e", "jump disp32 bytes away if lesser or equal (signed), if ZF is set or SF != OF (jcc/jle/jng)");
-put_new(Name_0f, "86", "jump disp32 bytes away if lesser or equal (unsigned), if ZF is set or CF is set (jcc/jbe/jna)");
+put_new(Name_0f, "8e", "jump disp32 bytes away if lesser or equal, if ZF is set or SF != OF (jcc/jle/jng)");
+put_new(Name_0f, "86", "jump disp32 bytes away if lesser or equal (addr, float), if ZF is set or CF is set (jcc/jbe/jna)");
 
 :(code)
 void test_jle_disp32_equal() {
diff --git a/023float.cc b/023float.cc
index cbd1bc80..324ab934 100644
--- a/023float.cc
+++ b/023float.cc
@@ -375,3 +375,42 @@ float* effective_address_float(uint8_t modrm) {
   trace(Callstack_depth+1, "run") << "effective address contains " << read_mem_f32(addr) << end();
   return mem_addr_f32(addr);
 }
+
+//: compare
+
+:(before "End Initialize Op Names")
+put_new(Name_0f, "2f", "compare: set SF if x32 < xm32 (comiss)");
+
+:(code)
+void test_compare_x32_with_mem_at_rm32() {
+  Reg[EAX].i = 0x2000;
+  Xmm[3] = 0.5;
+  run(
+      "== code 0x1\n"
+      // op     ModR/M  SIB   displacement  immediate
+      "  0f 2f  18                                    \n"  // compare XMM3 with *EAX
+      // ModR/M in binary: 00 (indirect mode) 011 (lhs XMM3) 000 (rhs EAX)
+      "== data 0x2000\n"
+      "00 00 00 00\n"  // 0x00000000 = 0.0
+  );
+  CHECK_TRACE_CONTENTS(
+      "run: compare XMM3 with x/m32\n"
+      "run: effective address is 0x00002000 (EAX)\n"
+      "run: SF=0; ZF=0; CF=0; OF=0\n"
+  );
+}
+
+:(before "End Two-Byte Opcodes Starting With 0f")
+case 0x2f: {  // set SF if x32 < x/m32
+  const uint8_t modrm = next();
+  const uint8_t reg1 = (modrm>>3)&0x7;
+  trace(Callstack_depth+1, "run") << "compare " << Xname[reg1] << " with x/m32" << end();
+  const float* arg2 = effective_address_float(modrm);
+  // Flag settings carefully copied from the Intel manual.
+  // See also https://stackoverflow.com/questions/7057501/x86-assembler-floating-point-compare/7057771#7057771
+  SF = ZF = CF = OF = false;
+  if (Xmm[reg1] == *arg2) ZF = true;
+  if (Xmm[reg1] < *arg2) CF = true;
+  trace(Callstack_depth+1, "run") << "SF=" << SF << "; ZF=" << ZF << "; CF=" << CF << "; OF=" << OF << end();
+  break;
+}
diff --git a/033check_operands.cc b/033check_operands.cc
index 66f74e80..7089bbb2 100644
--- a/033check_operands.cc
+++ b/033check_operands.cc
@@ -677,7 +677,8 @@ put_new(Permitted_arguments_0f, "8f", 0x10);
 //// Class M: using ModR/M byte
 //  imm32 imm8  disp32 |disp16  disp8 subop modrm
 //  0     0     0      |0       0     0     1
-put_new(Permitted_arguments_0f, "af", 0x01);
+put_new(Permitted_arguments_0f, "2f", 0x01);  // compare floats
+put_new(Permitted_arguments_0f, "af", 0x01);  // multiply ints
 // setcc
 put_new(Permitted_arguments_0f, "92", 0x01);
 put_new(Permitted_arguments_0f, "93", 0x01);