2581 - make space for the refcount in address:shared

We don't yet actually maintain the refcount. That's next.

Hardest part of this was debugging the assume-console scenarios in layer
85. That took some detailed manual diffing of traces (because the output
of diff was no good).

New tracing added in this commit add 8% to .traces LoC. Commented out
trace() calls (used during debugging) make that 45%.

7 files changed, 97 insertions, 53 deletions

diff --git a/032array.cc b/032array.cc
index dbc6a4dc..f2eacb03 100644
--- a/032array.cc
+++ b/032array.cc
@@ -112,7 +112,7 @@ if (r.type && r.type->value == get(Type_ordinal, "array")) {
     raise_error << maybe(current_recipe_name()) << "'" << r.original_string << "' is an array of what?\n" << end();
     return 1;
   }
-  // skip the 'array' type to get at the element type
+//?   trace(9999, "mem") << "computing size of array starting at " << r.value << end();
   return 1 + get_or_insert(Memory, r.value)*size_of(array_element(r.type));
 }
 
@@ -171,6 +171,7 @@ case INDEX: {
   reagent base = current_instruction().ingredients.at(0);
   canonize(base);
   long long int base_address = base.value;
+  trace(9998, "run") << "base address is " << base_address << end();
   if (base_address == 0) {
     raise_error << maybe(current_recipe_name()) << "tried to access location 0 in '" << current_instruction().to_string() << "'\n" << end();
     break;
diff --git a/038new.cc b/038new.cc
index d0e4f988..b49e347e 100644
--- a/038new.cc
+++ b/038new.cc
@@ -35,7 +35,8 @@ type_ordinal shared = put(Type_ordinal, "shared", Next_type_ordinal++);
 get_or_insert(Type, shared).name = "shared";
 :(before "End Drop Address In lookup_memory(x)")
 if (x.properties.at(0).second->value == "shared") {
-  //x.set_value(x.value+1);  // doesn't work yet
+  trace(9999, "mem") << "skipping refcount at " << x.value << end();
+  x.set_value(x.value+1);  // skip refcount
   drop_from_type(x, "shared");
 }
 :(before "End Drop Address In canonize_type(r)")
@@ -149,6 +150,9 @@ case ALLOCATE: {
     trace(9999, "mem") << "array size is " << ingredients.at(1).at(0) << end();
     size = /*space for length*/1 + size*ingredients.at(1).at(0);
   }
+  // include space for refcount
+  size++;
+  trace(9999, "mem") << "allocating size " << size << end();
 //?   Total_alloc += size;
 //?   Num_alloc++;
   // compute the region of memory to return
@@ -163,8 +167,10 @@ case ALLOCATE: {
   for (long long int address = result; address < result+size; ++address)
     put(Memory, address, 0);
   // initialize array length
-  if (SIZE(current_instruction().ingredients) > 1)
-    put(Memory, result, ingredients.at(1).at(0));
+  if (SIZE(current_instruction().ingredients) > 1) {
+    trace(9999, "mem") << "storing " << ingredients.at(1).at(0) << " in location " << result+/*skip refcount*/1 << end();
+    put(Memory, result+/*skip refcount*/1, ingredients.at(1).at(0));
+  }
   // bump
   Current_routine->alloc += size;
   // no support for reclaiming memory
@@ -202,7 +208,7 @@ case NEW: {
 void ensure_space(long long int size) {
   if (size > Initial_memory_per_routine) {
     tb_shutdown();
-    cerr << "can't allocate " << size << " locations, that's too much.\n";
+    cerr << "can't allocate " << size << " locations, that's too much compared to " << Initial_memory_per_routine << ".\n";
     exit(0);
   }
   if (Current_routine->alloc + size > Current_routine->alloc_max) {
@@ -231,8 +237,8 @@ recipe main [
 ]
 +run: 1:address:shared:array:number/raw <- new number:type, 5
 +mem: array size is 5
-# don't forget the extra location for array size
-+mem: storing 6 in location 3
+# don't forget the extra location for array size, and the second extra location for the refcount
++mem: storing 7 in location 3
 
 :(scenario new_empty_array)
 recipe main [
@@ -242,17 +248,18 @@ recipe main [
 ]
 +run: 1:address:shared:array:number/raw <- new number:type, 0
 +mem: array size is 0
-+mem: storing 1 in location 3
+# one location for array size, and one for the refcount
++mem: storing 2 in location 3
 
 //: If a routine runs out of its initial allocation, it should allocate more.
 :(scenario new_overflow)
-% Initial_memory_per_routine = 2;
+% Initial_memory_per_routine = 3;  // barely enough room for point allocation below
 recipe main [
   1:address:shared:number/raw <- new number:type
   2:address:shared:point/raw <- new point:type  # not enough room in initial page
 ]
-+new: routine allocated memory from 1000 to 1002
-+new: routine allocated memory from 1002 to 1004
++new: routine allocated memory from 1000 to 1003
++new: routine allocated memory from 1003 to 1006
 
 //: We also provide a way to return memory, and to reuse reclaimed memory.
 //: todo: custodians, etc. Following malloc/free is a temporary hack.
@@ -293,19 +300,23 @@ case ABANDON: {
 :(before "End Primitive Recipe Implementations")
 case ABANDON: {
   long long int address = ingredients.at(0).at(0);
+  trace(9999, "abandon") << "address to abandon is " << address << end();
   reagent types = current_instruction().ingredients.at(0);
+  trace(9999, "abandon") << "value of ingredient is " << types.value << end();
   canonize(types);
   // lookup_memory without drop_one_lookup {
-  types.set_value(get_or_insert(Memory, types.value));
+  trace(9999, "abandon") << "value of ingredient after canonization is " << types.value << end();
+  types.set_value(get_or_insert(Memory, types.value)+/*skip refcount*/1);
   drop_from_type(types, "address");
   drop_from_type(types, "shared");
   // }
-  abandon(address, size_of(types));
+  abandon(address, size_of(types)+/*refcount*/1);
   break;
 }
 
 :(code)
 void abandon(long long int address, long long int size) {
+  trace(9999, "abandon") << "saving in free-list of size " << size << end();
 //?   Total_free += size;
 //?   Num_free++;
 //?   cerr << "abandon: " << size << '\n';
@@ -319,6 +330,7 @@ void abandon(long long int address, long long int size) {
 
 :(before "ensure_space(size)" following "case ALLOCATE")
 if (Free_list[size]) {
+  trace(9999, "abandon") << "picking up space from free-list of size " << size << end();
   long long int result = Free_list[size];
   Free_list[size] = get_or_insert(Memory, result);
   for (long long int curr = result+1; curr < result+size; ++curr) {
@@ -328,7 +340,7 @@ if (Free_list[size]) {
     }
   }
   if (SIZE(current_instruction().ingredients) > 1)
-    put(Memory, result, ingredients.at(1).at(0));
+    put(Memory, result+/*skip refcount*/1, ingredients.at(1).at(0));
   else
     put(Memory, result, 0);
   products.resize(1);
@@ -396,6 +408,9 @@ long long int new_mu_string(const string& contents) {
   ensure_space(string_length+1);  // don't forget the extra location for array size
   // initialize string
   long long int result = Current_routine->alloc;
+  // initialize refcount
+  put(Memory, Current_routine->alloc++, 0);
+  // store length
   put(Memory, Current_routine->alloc++, string_length);
   long long int curr = 0;
   const char* raw_contents = contents.c_str();
@@ -465,6 +480,8 @@ long long int unicode_length(const string& s) {
 }
 
 string read_mu_string(long long int address) {
+  if (address == 0) return "";
+  address++;  // skip refcount
   long long int size = get_or_insert(Memory, address);
   if (size == 0) return "";
   ostringstream tmp;
diff --git a/043space.cc b/043space.cc
index 358b941a..662e7527 100644
--- a/043space.cc
+++ b/043space.cc
@@ -3,21 +3,24 @@
 //: (unless they have the /raw property)
 
 :(scenario set_default_space)
-# if default-space is 10, and if an array of 5 locals lies from location 11 to 15 (inclusive),
-# then location 0 is really location 11, location 1 is really location 12, and so on.
+# if default-space is 10, and if an array of 5 locals lies from location 12 to 16 (inclusive),
+# then local 0 is really location 12, local 1 is really location 13, and so on.
 recipe main [
-  10:number <- copy 5  # pretend array; in practice we'll use new
+  # pretend shared:array:location; in practice we'll use new
+  10:number <- copy 0  # refcount
+  11:number <- copy 5  # length
   default-space:address:shared:array:location <- copy 10/unsafe
   1:number <- copy 23
 ]
-+mem: storing 23 in location 12
++mem: storing 23 in location 13
 
 :(scenario lookup_sidesteps_default_space)
 recipe main [
   # pretend pointer from outside
   3:number <- copy 34
-  # pretend array
-  1000:number <- copy 5
+  # pretend shared:array:location; in practice we'll use new
+  1000:number <- copy 0  # refcount
+  1001:number <- copy 5  # length
   # actual start of this recipe
   default-space:address:shared:array:location <- copy 1000/unsafe
   1:address:number <- copy 3/unsafe
@@ -62,7 +65,7 @@ void absolutize(reagent& x) {
 
 long long int space_base(const reagent& x) {
   // temporary stub; will be replaced in a later layer
-  return current_call().default_space;
+  return current_call().default_space ? (current_call().default_space+/*skip refcount*/1) : 0;
 }
 
 long long int address(long long int offset, long long int base) {
@@ -117,8 +120,9 @@ recipe main [
   # pretend pointer to container from outside
   12:number <- copy 34
   13:number <- copy 35
-  # pretend array
-  1000:number <- copy 5
+  # pretend shared:array:location; in practice we'll use new
+  1000:number <- copy 0  # refcount
+  1001:number <- copy 5  # length
   # actual start of this recipe
   default-space:address:shared:array:location <- copy 1000/unsafe
   1:address:point <- copy 12/unsafe
@@ -137,8 +141,9 @@ recipe main [
   12:number <- copy 2
   13:number <- copy 34
   14:number <- copy 35
-  # pretend array
-  1000:number <- copy 5
+  # pretend shared:array:location; in practice we'll use new
+  1000:number <- copy 0  # refcount
+  1001:number <- copy 5  # length
   # actual start of this recipe
   default-space:address:shared:array:location <- copy 1000/unsafe
   1:address:array:number <- copy 12/unsafe
@@ -225,7 +230,7 @@ void try_reclaim_locals() {
   const instruction& inst = get(Recipe, r).steps.at(0);
   if (inst.old_name != "local-scope") return;
   abandon(current_call().default_space,
-          /*array length*/1+/*number-of-locals*/Name[r][""]);
+          /*refcount*/1 + /*array length*/1 + /*number-of-locals*/Name[r][""]);
 }
 
 void rewrite_default_space_instruction(instruction& curr) {
diff --git a/044space_surround.cc b/044space_surround.cc
index d645f1b4..684a69af 100644
--- a/044space_surround.cc
+++ b/044space_surround.cc
@@ -7,21 +7,26 @@
 :(scenario surrounding_space)
 # location 1 in space 1 refers to the space surrounding the default space, here 20.
 recipe main [
-  10:number <- copy 5  # pretend array
-  20:number <- copy 5  # pretend array
+  # pretend shared:array:location; in practice we'll use new
+  10:number <- copy 0  # refcount
+  11:number <- copy 5  # length
+  # pretend shared:array:location; in practice we'll use new
+  20:number <- copy 0  # refcount
+  21:number <- copy 5  # length
+  # actual start of this recipe
   default-space:address:shared:array:location <- copy 10/unsafe
   0:address:shared:array:location/names:dummy <- copy 20/unsafe  # later layers will explain the /names: property
   1:number <- copy 32
   1:number/space:1 <- copy 33
 ]
-recipe dummy [
+recipe dummy [  # just for the /names: property above
 ]
-# chain space
-+mem: storing 20 in location 11
-# store to default-space
-+mem: storing 32 in location 12
-# store to chained space
-+mem: storing 33 in location 22
+# chain space: 10 + /*skip refcount*/1 + /*skip length*/1
++mem: storing 20 in location 12
+# store to default space: 10 + /*skip refcount*/1 + /*skip length*/1 + /*index*/1
++mem: storing 32 in location 13
+# store to chained space: /*contents of location 12*/20 + /*skip refcount*/1 + /*skip length*/1 + /*index*/1
++mem: storing 33 in location 23
 
 //: If you think of a space as a collection of variables with a common
 //: lifetime, surrounding allows managing shorter lifetimes inside a longer
@@ -29,14 +34,17 @@ recipe dummy [
 
 :(replace{} "long long int space_base(const reagent& x)")
 long long int space_base(const reagent& x) {
-  return space_base(x, space_index(x), current_call().default_space);
+  long long int base = current_call().default_space ? (current_call().default_space+/*skip refcount*/1) : 0;
+  return space_base(x, space_index(x), base);
 }
 
 long long int space_base(const reagent& x, long long int space_index, long long int base) {
+//?   trace(9999, "space") << "space-base: " << space_index << " " << base << end();
   if (space_index == 0) {
     return base;
   }
-  long long int result = space_base(x, space_index-1, get_or_insert(Memory, base+1));
+  long long int result = space_base(x, space_index-1, get_or_insert(Memory, base+/*skip length*/1))+/*skip refcount*/1;
+//?   trace(9999, "space") << "space-base: " << space_index << " " << base << " => " << result << end();
   return result;
 }
 
diff --git a/046global.cc b/046global.cc
index 3393ed31..87a39937 100644
--- a/046global.cc
+++ b/046global.cc
@@ -5,17 +5,22 @@
 
 :(scenario global_space)
 recipe main [
-  # pretend arrays; in practice we'll use new
-  10:number <- copy 5
-  20:number <- copy 5
+  # pretend shared:array:location; in practice we'll use new
+  10:number <- copy 0  # refcount
+  11:number <- copy 5  # length
+  # pretend shared:array:location; in practice we'll use new
+  20:number <- copy 0  # refcount
+  21:number <- copy 5  # length
   # actual start of this recipe
   global-space:address:shared:array:location <- copy 20/unsafe
   default-space:address:shared:array:location <- copy 10/unsafe
   1:number <- copy 23
   1:number/space:global <- copy 24
 ]
-+mem: storing 23 in location 12
-+mem: storing 24 in location 22
+# store to default space: 10 + /*skip refcount*/1 + /*skip length*/1 + /*index*/1
++mem: storing 23 in location 13
+# store to chained space: /*contents of location 12*/20 + /*skip refcount*/1 + /*skip length*/1 + /*index*/1
++mem: storing 24 in location 23
 
 //: to support it, create another special variable called global space
 :(before "End Disqualified Reagents")
@@ -54,7 +59,7 @@ global_space = 0;
   if (is_global(x)) {
     if (!Current_routine->global_space)
       raise_error << "routine has no global space\n" << end();
-    return Current_routine->global_space;
+    return Current_routine->global_space + /*skip refcount*/1;
   }
 
 //: for now let's not bother giving global variables names.
diff --git a/082scenario_screen.cc b/082scenario_screen.cc
index 362c73f1..89562f6a 100644
--- a/082scenario_screen.cc
+++ b/082scenario_screen.cc
@@ -207,18 +207,18 @@ struct raw_string_stream {
 :(code)
 void check_screen(const string& expected_contents, const int color) {
   assert(!current_call().default_space);  // not supported
-  long long int screen_location = get_or_insert(Memory, SCREEN);
+  long long int screen_location = get_or_insert(Memory, SCREEN)+/*skip refcount*/1;
   int data_offset = find_element_name(get(Type_ordinal, "screen"), "data", "");
   assert(data_offset >= 0);
   long long int screen_data_location = screen_location+data_offset;  // type: address:shared:array:character
-  long long int screen_data_start = get_or_insert(Memory, screen_data_location);  // type: array:character
+  long long int screen_data_start = get_or_insert(Memory, screen_data_location) + /*skip refcount*/1;  // type: array:character
   int width_offset = find_element_name(get(Type_ordinal, "screen"), "num-columns", "");
   long long int screen_width = get_or_insert(Memory, screen_location+width_offset);
   int height_offset = find_element_name(get(Type_ordinal, "screen"), "num-rows", "");
   long long int screen_height = get_or_insert(Memory, screen_location+height_offset);
   raw_string_stream cursor(expected_contents);
   // todo: too-long expected_contents should fail
-  long long int addr = screen_data_start+1;  // skip length
+  long long int addr = screen_data_start+/*skip length*/1;
   for (long long int row = 0; row < screen_height; ++row) {
     cursor.skip_whitespace_and_comments();
     if (cursor.at_end()) break;
@@ -342,7 +342,7 @@ case _DUMP_SCREEN: {
 :(code)
 void dump_screen() {
   assert(!current_call().default_space);  // not supported
-  long long int screen_location = get_or_insert(Memory, SCREEN);
+  long long int screen_location = get_or_insert(Memory, SCREEN) + /*skip refcount*/1;
   int width_offset = find_element_name(get(Type_ordinal, "screen"), "num-columns", "");
   long long int screen_width = get_or_insert(Memory, screen_location+width_offset);
   int height_offset = find_element_name(get(Type_ordinal, "screen"), "num-rows", "");
@@ -350,7 +350,7 @@ void dump_screen() {
   int data_offset = find_element_name(get(Type_ordinal, "screen"), "data", "");
   assert(data_offset >= 0);
   long long int screen_data_location = screen_location+data_offset;  // type: address:shared:array:character
-  long long int screen_data_start = get_or_insert(Memory, screen_data_location);  // type: array:character
+  long long int screen_data_start = get_or_insert(Memory, screen_data_location) + /*skip refcount*/1;  // type: array:character
   assert(get_or_insert(Memory, screen_data_start) == screen_width*screen_height);
   long long int curr = screen_data_start+1;  // skip length
   for (long long int row = 0; row < screen_height; ++row) {
diff --git a/085scenario_console.cc b/085scenario_console.cc
index 25737c85..1b746adb 100644
--- a/085scenario_console.cc
+++ b/085scenario_console.cc
@@ -54,14 +54,16 @@ case ASSUME_CONSOLE: {
   slurp_body(in, r);
   long long int num_events = count_events(r);
   // initialize the events like in new-fake-console
-  long long int size = num_events*size_of_event() + /*space for length*/1;
+  long long int size = /*space for refcount*/1 + /*space for length*/1 + num_events*size_of_event();
   ensure_space(size);
   long long int event_data_address = Current_routine->alloc;
-  put(Memory, event_data_address, num_events);
-  ++Current_routine->alloc;
+  // store length
+  put(Memory, Current_routine->alloc+/*skip refcount*/1, num_events);
+  Current_routine->alloc += /*skip refcount and length*/2;
   for (long long int i = 0; i < SIZE(r.steps); ++i) {
     const instruction& curr = r.steps.at(i);
     if (curr.name == "left-click") {
+      trace(9999, "mem") << "storing 'left-click' event starting at " << Current_routine->alloc << end();
       put(Memory, Current_routine->alloc, /*tag for 'touch-event' variant of 'event' exclusive-container*/2);
       put(Memory, Current_routine->alloc+1+/*offset of 'type' in 'mouse-event'*/0, TB_KEY_MOUSE_LEFT);
       put(Memory, Current_routine->alloc+1+/*offset of 'row' in 'mouse-event'*/1, to_integer(curr.ingredients.at(0).name));
@@ -69,6 +71,7 @@ case ASSUME_CONSOLE: {
       Current_routine->alloc += size_of_event();
     }
     else if (curr.name == "press") {
+      trace(9999, "mem") << "storing 'press' event starting at " << Current_routine->alloc << end();
       string key = curr.ingredients.at(0).name;
       if (is_integer(key))
         put(Memory, Current_routine->alloc+1, to_integer(key));
@@ -89,15 +92,18 @@ case ASSUME_CONSOLE: {
     else {
       // keyboard input
       assert(curr.name == "type");
+      trace(9999, "mem") << "storing 'type' event starting at " << Current_routine->alloc << end();
       const string& contents = curr.ingredients.at(0).name;
       const char* raw_contents = contents.c_str();
       long long int num_keyboard_events = unicode_length(contents);
       long long int curr = 0;
       for (long long int i = 0; i < num_keyboard_events; ++i) {
+        trace(9999, "mem") << "storing 'text' tag at " << Current_routine->alloc << end();
         put(Memory, Current_routine->alloc, /*tag for 'text' variant of 'event' exclusive-container*/0);
         uint32_t curr_character;
         assert(curr < SIZE(contents));
         tb_utf8_char_to_unicode(&curr_character, &raw_contents[curr]);
+        trace(9999, "mem") << "storing character " << curr_character << " at " << Current_routine->alloc+1 << end();
         put(Memory, Current_routine->alloc+/*skip exclusive container tag*/1, curr_character);
         curr += tb_utf8_char_length(raw_contents[curr]);
         Current_routine->alloc += size_of_event();
@@ -108,9 +114,11 @@ case ASSUME_CONSOLE: {
   // wrap the array of events in a console object
   ensure_space(size_of_console());
   put(Memory, CONSOLE, Current_routine->alloc);
+  trace(9999, "mem") << "storing console in " << Current_routine->alloc << end();
   Current_routine->alloc += size_of_console();
   long long int console_address = get_or_insert(Memory, CONSOLE);
-  put(Memory, console_address+/*offset of 'data' in container 'events'*/1, event_data_address);
+  trace(9999, "mem") << "storing console data in " << console_address+2 << end();
+  put(Memory, console_address+/*skip refcount*/1+/*offset of 'data' in container 'events'*/1, event_data_address);
   break;
 }
 
@@ -281,7 +289,7 @@ long long int size_of_console() {
   if (result) return result;
   assert(get(Type_ordinal, "console"));
   type_tree* type = new type_tree(get(Type_ordinal, "console"));
-  result = size_of(type);
+  result = size_of(type)+/*refcount*/1;
   delete type;
   return result;
 }