5152 - check for stack underflow/overflow in VM

6 files changed, 33 insertions, 22 deletions

diff --git a/subx/012elf.cc b/subx/012elf.cc
index d0a3fbd2..a77c6056 100644
--- a/subx/012elf.cc
+++ b/subx/012elf.cc
@@ -90,6 +90,7 @@ void load_elf_contents(uint8_t* elf_contents, size_t size, int argc, char* argv[
 
 void push(uint32_t val) {
   Reg[ESP].u -= 4;
+  assert(Reg[ESP].u >= STACK_SEGMENT);
   trace(Callstack_depth+1, "run") << "decrementing ESP to 0x" << HEXWORD << Reg[ESP].u << end();
   trace(Callstack_depth+1, "run") << "pushing value 0x" << HEXWORD << val << end();
   write_mem_u32(Reg[ESP].u, val);
diff --git a/subx/013direct_addressing.cc b/subx/013direct_addressing.cc
index c2dfa911..160ce6d6 100644
--- a/subx/013direct_addressing.cc
+++ b/subx/013direct_addressing.cc
@@ -971,7 +971,8 @@ put_new(Name, "57", "push EDI to stack (push)");
 
 :(code)
 void test_push_r32() {
-  Reg[ESP].u = 0x64;
+  Mem.push_back(vma(0x7d000000));  // manually allocate memory
+  Reg[ESP].u = 0x7d000008;
   Reg[EBX].i = 0x0000000a;
   run(
       "== 0x1\n"  // code segment
@@ -980,7 +981,7 @@ void test_push_r32() {
   );
   CHECK_TRACE_CONTENTS(
       "run: push EBX\n"
-      "run: decrementing ESP to 0x00000060\n"
+      "run: decrementing ESP to 0x7d000004\n"
       "run: pushing value 0x0000000a\n"
   );
 }
@@ -1015,9 +1016,9 @@ put_new(Name, "5f", "pop top of stack to EDI (pop)");
 
 :(code)
 void test_pop_r32() {
-  Reg[ESP].u = 0x02000000;
-  Mem.push_back(vma(0x02000000));  // manually allocate memory
-  write_mem_i32(0x02000000, 0x0000000a);  // ..before this write
+  Mem.push_back(vma(0x7d000000));  // manually allocate memory
+  Reg[ESP].u = 0x7d000008;
+  write_mem_i32(0x7d000008, 0x0000000a);  // ..before this write
   run(
       "== 0x1\n"  // code segment
       // op     ModR/M  SIB   displacement  immediate
@@ -1028,7 +1029,7 @@ void test_pop_r32() {
   CHECK_TRACE_CONTENTS(
       "run: pop into EBX\n"
       "run: popping value 0x0000000a\n"
-      "run: incrementing ESP to 0x02000004\n"
+      "run: incrementing ESP to 0x7d00000c\n"
   );
 }
 
@@ -1054,5 +1055,6 @@ uint32_t pop() {
   trace(Callstack_depth+1, "run") << "popping value 0x" << HEXWORD << result << end();
   Reg[ESP].u += 4;
   trace(Callstack_depth+1, "run") << "incrementing ESP to 0x" << HEXWORD << Reg[ESP].u << end();
+  assert(Reg[ESP].u < AFTER_STACK);
   return result;
 }
diff --git a/subx/014indirect_addressing.cc b/subx/014indirect_addressing.cc
index f591f0cf..8f0d3325 100644
--- a/subx/014indirect_addressing.cc
+++ b/subx/014indirect_addressing.cc
@@ -531,7 +531,8 @@ case 4: {  // jump to r/m32
 :(code)
 void test_push_mem_at_r32() {
   Reg[EAX].i = 0x2000;
-  Reg[ESP].u = 0x14;
+  Mem.push_back(vma(0x7d000000));  // manually allocate memory
+  Reg[ESP].u = 0x7d000014;
   run(
       "== 0x1\n"  // code segment
       // op     ModR/M  SIB   displacement  immediate
@@ -542,7 +543,7 @@ void test_push_mem_at_r32() {
   CHECK_TRACE_CONTENTS(
       "run: push r/m32\n"
       "run: effective address is 0x00002000 (EAX)\n"
-      "run: decrementing ESP to 0x00000010\n"
+      "run: decrementing ESP to 0x7d000010\n"
       "run: pushing value 0x000000af\n"
   );
 }
@@ -563,20 +564,20 @@ put_new(Name, "8f", "pop top of stack to rm32 (pop)");
 :(code)
 void test_pop_mem_at_r32() {
   Reg[EAX].i = 0x60;
-  Reg[ESP].u = 0x2000;
+  Mem.push_back(vma(0x7d000000));  // manually allocate memory
+  Reg[ESP].u = 0x7d000000;
+  write_mem_i32(0x7d000000, 0x00000030);
   run(
       "== 0x1\n"  // code segment
       // op     ModR/M  SIB   displacement  immediate
       "  8f     00                                    \n"  // pop stack into *EAX
       // ModR/M in binary: 00 (indirect mode) 000 (pop r/m32) 000 (dest EAX)
-      "== 0x2000\n"  // data segment
-      "30 00 00 00\n"  // 0x00000030
   );
   CHECK_TRACE_CONTENTS(
       "run: pop into r/m32\n"
       "run: effective address is 0x00000060 (EAX)\n"
       "run: popping value 0x00000030\n"
-      "run: incrementing ESP to 0x00002004\n"
+      "run: incrementing ESP to 0x7d000004\n"
   );
 }
 
diff --git a/subx/015immediate_addressing.cc b/subx/015immediate_addressing.cc
index 18cd5334..16d886e8 100644
--- a/subx/015immediate_addressing.cc
+++ b/subx/015immediate_addressing.cc
@@ -824,7 +824,8 @@ put_new(Name, "68", "push imm32 to stack (push)");
 
 :(code)
 void test_push_imm32() {
-  Reg[ESP].u = 0x14;
+  Mem.push_back(vma(0x7d000000));  // manually allocate memory
+  Reg[ESP].u = 0x7d000014;
   run(
       "== 0x1\n"  // code segment
       // op     ModR/M  SIB   displacement  immediate
@@ -832,7 +833,7 @@ void test_push_imm32() {
   );
   CHECK_TRACE_CONTENTS(
       "run: push imm32 0x000000af\n"
-      "run: ESP is now 0x00000010\n"
+      "run: ESP is now 0x7d000010\n"
       "run: contents at ESP: 0x000000af\n"
   );
 }
diff --git a/subx/019functions.cc b/subx/019functions.cc
index 7f45167b..27fb4fb0 100644
--- a/subx/019functions.cc
+++ b/subx/019functions.cc
@@ -5,7 +5,8 @@ put_new(Name, "e8", "call disp32 (call)");
 
 :(code)
 void test_call_disp32() {
-  Reg[ESP].u = 0x64;
+  Mem.push_back(vma(0x7d000000));  // manually allocate memory
+  Reg[ESP].u = 0x7d000064;
   run(
       "== 0x1\n"  // code segment
       // op     ModR/M  SIB   displacement  immediate
@@ -14,7 +15,7 @@ void test_call_disp32() {
   );
   CHECK_TRACE_CONTENTS(
       "run: call imm32 0x000000a0\n"
-      "run: decrementing ESP to 0x00000060\n"
+      "run: decrementing ESP to 0x7d000060\n"
       "run: pushing value 0x00000006\n"
       "run: jumping to 0x000000a6\n"
   );
@@ -36,7 +37,8 @@ case 0xe8: {  // call disp32 relative to next EIP
 
 :(code)
 void test_call_r32() {
-  Reg[ESP].u = 0x64;
+  Mem.push_back(vma(0x7d000000));  // manually allocate memory
+  Reg[ESP].u = 0x7d000064;
   Reg[EBX].u = 0x000000a0;
   run(
       "== 0x1\n"  // code segment
@@ -47,7 +49,7 @@ void test_call_r32() {
   CHECK_TRACE_CONTENTS(
       "run: call to r/m32\n"
       "run: r/m32 is EBX\n"
-      "run: decrementing ESP to 0x00000060\n"
+      "run: decrementing ESP to 0x7d000060\n"
       "run: pushing value 0x00000003\n"
       "run: jumping to 0x000000a3\n"
   );
@@ -66,7 +68,8 @@ case 2: {  // call function pointer at r/m32
 
 :(code)
 void test_call_mem_at_r32() {
-  Reg[ESP].u = 0x64;
+  Mem.push_back(vma(0x7d000000));  // manually allocate memory
+  Reg[ESP].u = 0x7d000064;
   Reg[EBX].u = 0x2000;
   run(
       "== 0x1\n"  // code segment
@@ -79,7 +82,7 @@ void test_call_mem_at_r32() {
   CHECK_TRACE_CONTENTS(
       "run: call to r/m32\n"
       "run: effective address is 0x00002000 (EBX)\n"
-      "run: decrementing ESP to 0x00000060\n"
+      "run: decrementing ESP to 0x7d000060\n"
       "run: pushing value 0x00000003\n"
       "run: jumping to 0x000000a3\n"
   );
@@ -92,7 +95,9 @@ put_new(Name, "c3", "return from most recent unfinished call (ret)");
 
 :(code)
 void test_ret() {
-  Reg[ESP].u = 0x2000;
+  Mem.push_back(vma(0x7d000000));  // manually allocate memory
+  Reg[ESP].u = 0x7d000064;
+  write_mem_u32(Reg[ESP].u, 0x10);
   run(
       "== 0x1\n"  // code segment
       // op     ModR/M  SIB   displacement  immediate
diff --git a/subx/040---tests.cc b/subx/040---tests.cc
index d35cc711..237bb811 100644
--- a/subx/040---tests.cc
+++ b/subx/040---tests.cc
@@ -16,7 +16,8 @@ Transform.push_back(create_test_function);
 
 :(code)
 void test_run_test() {
-  Reg[ESP].u = 0x100;
+  Mem.push_back(vma(0x7d000000));  // manually allocate memory
+  Reg[ESP].u = 0x7d000100;
   run(
       "== 0x1\n"  // code segment
       "main:\n"