2898 - start filling in missing refcounts

This commit covers instructions 'put', 'put-index' and 'maybe-convert'.
Next up are the harder ones: 'copy' and 'merge'. In these cases there's
a non-scalar being copied, and we need to figure out which locations
within it need to update their refcount.

7 files changed, 110 insertions, 22 deletions

diff --git a/030container.cc b/030container.cc
index 071cc461..8018cb91 100644
--- a/030container.cc
+++ b/030container.cc
@@ -425,12 +425,11 @@ case PUT: {
   type_ordinal base_type = base.type->value;
   int offset = ingredients.at(1).at(0);
   if (offset < 0 || offset >= SIZE(get(Type, base_type).elements)) break;  // copied from Check above
-  int address = base_address;
-  for (int i = 0; i < offset; ++i)
-    address += size_of(element_type(base.type, i));
+  int address = base_address + base.metadata.offset.at(offset);
   trace(9998, "run") << "address to copy to is " << address << end();
   // optimization: directly write the element rather than updating 'product'
   // and writing the entire container
+  // Write Memory in PUT in Run
   for (int i = 0; i < SIZE(ingredients.at(2)); ++i) {
     trace(9999, "mem") << "storing " << no_scientific(ingredients.at(2).at(i)) << " in location " << address+i << end();
     put(Memory, address+i, ingredients.at(2).at(i));
diff --git a/032array.cc b/032array.cc
index a40f9e20..3d93e84d 100644
--- a/032array.cc
+++ b/032array.cc
@@ -361,13 +361,15 @@ case PUT_INDEX: {
     raise << maybe(current_recipe_name()) << "invalid index " << no_scientific(index_val.at(0)) << '\n' << end();
     break;
   }
-  type_tree* element_type = copy_array_element(base.type);
-  int address = base_address + 1 + index_val.at(0)*size_of(element_type);
-  delete element_type;
+  reagent element;
+  element.type = copy_array_element(base.type);
+  int address = base_address + 1 + index_val.at(0)*size_of(element.type);
+  element.value = address;
   trace(9998, "run") << "address to copy to is " << address << end();
   // optimization: directly write the element rather than updating 'product'
   // and writing the entire array
   vector<double> value = read_memory(current_instruction().ingredients.at(2));
+  // Write Memory in PUT_INDEX in Run
   for (int i = 0; i < SIZE(value); ++i) {
     trace(9999, "mem") << "storing " << no_scientific(value.at(i)) << " in location " << address+i << end();
     put(Memory, address+i, value.at(i));
diff --git a/033exclusive_container.cc b/033exclusive_container.cc
index e071cf52..e41e3f34 100644
--- a/033exclusive_container.cc
+++ b/033exclusive_container.cc
@@ -70,11 +70,11 @@ def main [
   14:number <- copy 36
   20:point, 22:boolean <- maybe-convert 12:number-or-point/unsafe, 1:variant
 ]
+# boolean
++mem: storing 1 in location 22
 # point
 +mem: storing 35 in location 20
 +mem: storing 36 in location 21
-# boolean
-+mem: storing 1 in location 22
 
 :(scenario maybe_convert_fail)
 def main [
@@ -83,9 +83,9 @@ def main [
   14:number <- copy 36
   20:number, 21:boolean <- maybe-convert 12:number-or-point/unsafe, 0:variant
 ]
-# number: no write
 # boolean
 +mem: storing 0 in location 21
+# number: no write
 
 :(before "End Primitive Recipe Declarations")
 MAYBE_CONVERT,
@@ -151,13 +151,14 @@ case MAYBE_CONVERT: {
   // optimization: directly write results to only update first product when necessary
   if (tag == static_cast<int>(get_or_insert(Memory, base_address))) {
     const reagent variant = variant_type(base, tag);
+    trace(9999, "mem") << "storing 1 in location " << status.value << end();
+    put(Memory, status.value, 1);
+    // Write Memory in Successful MAYBE_CONVERT in Run
     for (int i = 0; i < size_of(variant); ++i) {
-      double val = get_or_insert(Memory, base_address+1+i);
+      double val = get_or_insert(Memory, base_address+/*skip tag*/1+i);
       trace(9999, "mem") << "storing " << no_scientific(val) << " in location " << product.value+i << end();
       put(Memory, product.value+i, val);
     }
-    trace(9999, "mem") << "storing 1 in location " << status.value << end();
-    put(Memory, status.value, 1);
   }
   else {
     trace(9999, "mem") << "storing 0 in location " << status.value << end();
diff --git a/035lookup.cc b/035lookup.cc
index 2a7cb2e3..63897ba9 100644
--- a/035lookup.cc
+++ b/035lookup.cc
@@ -350,8 +350,8 @@ def main [
   1:address:number-or-point <- copy 10/unsafe
   2:number, 3:boolean <- maybe-convert 1:address:number-or-point/lookup, i:variant
 ]
-+mem: storing 34 in location 2
 +mem: storing 1 in location 3
++mem: storing 34 in location 2
 
 :(scenario maybe_convert_indirect_2)
 def main [
@@ -361,8 +361,8 @@ def main [
   2:address:number <- copy 20/unsafe
   2:address:number/lookup, 3:boolean <- maybe-convert 1:address:number-or-point/lookup, i:variant
 ]
-+mem: storing 34 in location 21
 +mem: storing 1 in location 3
++mem: storing 34 in location 21
 
 :(scenario maybe_convert_indirect_3)
 def main [
@@ -372,8 +372,8 @@ def main [
   2:address:boolean <- copy 20/unsafe
   3:number, 2:address:boolean/lookup <- maybe-convert 1:address:number-or-point/lookup, i:variant
 ]
-+mem: storing 34 in location 3
 +mem: storing 1 in location 21
++mem: storing 34 in location 3
 
 :(before "Update MAYBE_CONVERT base in Check")
 if (!canonize_type(base)) break;
diff --git a/036refcount.cc b/036refcount.cc
index a3ea72f9..939490ac 100644
--- a/036refcount.cc
+++ b/036refcount.cc
@@ -18,11 +18,17 @@ def main [
 +mem: decrementing refcount of 1000: 1 -> 0
 
 :(before "End write_memory(reagent x) Special-cases")
-if (x.type->value == get(Type_ordinal, "address")) {
+if (is_mu_address(x)) {
   // compute old address of x, as well as new address we want to write in
-  int old_address = get_or_insert(Memory, x.value);
   assert(scalar(data));
-  int new_address = data.at(0);
+  write_memory_updating_refcounts(x, data.at(0));
+  return;
+}
+:(code)
+// variant of write_memory for addresses
+void write_memory_updating_refcounts(const reagent& canonized_loc, int new_address) {
+  assert(is_mu_address(canonized_loc));
+  int old_address = get_or_insert(Memory, canonized_loc.value);
   // decrement refcount of old address
   if (old_address) {
     int old_refcount = get_or_insert(Memory, old_address);
@@ -30,8 +36,13 @@ if (x.type->value == get(Type_ordinal, "address")) {
     put(Memory, old_address, old_refcount-1);
   }
   // perform the write
-  trace(9999, "mem") << "storing " << no_scientific(data.at(0)) << " in location " << x.value << end();
-  put(Memory, x.value, new_address);
+  trace(9999, "mem") << "storing " << no_scientific(new_address) << " in location " << canonized_loc.value << end();
+  if (!canonized_loc.value) {
+    tb_shutdown();
+    DUMP("");
+    exit(0);
+  }
+  put(Memory, canonized_loc.value, new_address);
   // increment refcount of new address
   if (new_address) {
     int new_refcount = get_or_insert(Memory, new_address);
@@ -40,7 +51,6 @@ if (x.type->value == get(Type_ordinal, "address")) {
     put(Memory, new_address, new_refcount+1);
   }
   // End Update Reference Count
-  return;
 }
 
 :(scenario refcounts_reflexive)
@@ -72,3 +82,78 @@ def foo [
 +mem: incrementing refcount of 1000: 1 -> 2
 +run: {1: ("address" "number")} <- new {number: "type"}
 +mem: decrementing refcount of 1000: 2 -> 1
+
+//: fix up any instructions that don't follow the usual flow of read_memory
+//: before the RUN switch, and write_memory after
+
+:(scenario refcounts_put)
+container foo [
+  x:address:number
+]
+def main [
+  1:address:number <- new number:type
+  2:address:foo <- new foo:type
+  *2:address:foo <- put *2:address:foo, x:offset, 1:address:number
+]
++run: {1: ("address" "number")} <- new {number: "type"}
++mem: incrementing refcount of 1000: 0 -> 1
++run: {2: ("address" "foo")} <- new {foo: "type"}
++mem: incrementing refcount of 1002: 0 -> 1
++run: {2: ("address" "foo"), "lookup": ()} <- put {2: ("address" "foo"), "lookup": ()}, {x: "offset"}, {1: ("address" "number")}
+# put increments refcount
++mem: incrementing refcount of 1000: 1 -> 2
+
+:(after "Write Memory in PUT in Run")
+reagent element = element_type(base.type, offset);
+assert(!has_property(element, "lookup"));
+element.value = address;
+if (is_mu_address(element)) {
+  write_memory_updating_refcounts(element, ingredients.at(2).at(0));
+  goto finish_instruction;
+}
+
+:(scenario refcounts_put_index)
+def main [
+  1:address:number <- new number:type
+  # fake array because we can't yet create an array of addresses (wait for the
+  # support for dilated reagents and parsing more complex type trees)
+  1003:number/raw <- copy 3  # skip refcount at 1002
+  2:address:array:address:number <- copy 1002/unsafe
+  *2:address:array:address:number <- put-index *2:address:array:address:number, 0, 1:address:number
+]
++run: {1: ("address" "number")} <- new {number: "type"}
++mem: incrementing refcount of 1000: 0 -> 1
++run: {2: ("address" "array" "address" "number")} <- copy {1002: "literal", "unsafe": ()}
++mem: incrementing refcount of 1002: 0 -> 1
++run: {2: ("address" "array" "address" "number"), "lookup": ()} <- put-index {2: ("address" "array" "address" "number"), "lookup": ()}, {0: "literal"}, {1: ("address" "number")}
+# put-index increments refcount
++mem: incrementing refcount of 1000: 1 -> 2
+
+:(after "Write Memory in PUT_INDEX in Run")
+if (is_mu_address(element)) {
+  write_memory_updating_refcounts(element, value.at(0));
+  goto finish_instruction;
+}
+
+:(scenario refcounts_maybe_convert)
+exclusive-container foo [
+  x:number
+  p:address:number
+]
+def main [
+  1:address:number <- new number:type
+  2:foo <- merge 1/p, 1:address:number
+  4:address:number, 5:boolean <- maybe-convert 2:foo, p:variant
+]
++run: {1: ("address" "number")} <- new {number: "type"}
++mem: incrementing refcount of 1000: 0 -> 1
++run: {2: "foo"} <- merge {1: "literal", "p": ()}, {1: ("address" "number")}
++run: {4: ("address" "number")}, {5: "boolean"} <- maybe-convert {2: "foo"}, {p: "variant"}
+# maybe-convert increments refcount on success
++mem: incrementing refcount of 1000: 1 -> 2
+
+:(after "Write Memory in Successful MAYBE_CONVERT")
+if (is_mu_address(product)) {
+  write_memory_updating_refcounts(product, get_or_insert(Memory, base_address+/*skip tag*/1));
+  goto finish_instruction;
+}
diff --git a/037abandon.cc b/037abandon.cc
index f454123e..076d6e14 100644
--- a/037abandon.cc
+++ b/037abandon.cc
@@ -27,6 +27,7 @@ if (get_or_insert(Memory, old_address) > 0) return;
 // old_address has a 0 refcount
 // lookup_memory without drop_one_lookup {
 trace(9999, "mem") << "automatically abandoning " << old_address << end();
+reagent x = canonized_loc;
 trace(9999, "mem") << "computing size to abandon at " << x.value << end();
 x.set_value(old_address+/*skip refcount*/1);
 drop_from_type(x, "address");
diff --git a/042name.cc b/042name.cc
index 42011052..583cd1dc 100644
--- a/042name.cc
+++ b/042name.cc
@@ -269,9 +269,9 @@ def main [
   20:point, 22:boolean <- maybe-convert 12:number-or-point/unsafe, p:variant
 ]
 +name: variant p of type number-or-point has tag 1
++mem: storing 1 in location 22
 +mem: storing 35 in location 20
 +mem: storing 36 in location 21
-+mem: storing 1 in location 22
 
 :(before "End transform_names(inst) Special-cases")
 // convert variant names of exclusive containers