three bugs fixed

- notes
bug in edit/ triggers in immutable but not master branch
bug triggered by changes to layer 059: we're finding an unspecialized call to 'length' in 'append_6'

hard to debug because trace isn't complete
just bring out the big hammer: use a new log file

length_2 from recipes.mu is not being deleted (bug #1)
so reload doesn't switch length to length_2 when variant_already_exists (bug #2)
so we end up saving in Recipe for a primitive ordinal
so no valid specialization is found for 'length' (bug #3)

why doesn't it trigger in a non-interactive scenario?
argh, wasn't checking for an empty line at end. ok, confidence restored.

8 files changed, 253 insertions, 31 deletions

diff --git a/011load.cc b/011load.cc
index a4ae7de2..59d667a9 100644
--- a/011load.cc
+++ b/011load.cc
@@ -61,6 +61,7 @@ long long int slurp_recipe(istream& in) {
   // End recipe Body(result)
   get_or_insert(Recipe, get(Recipe_ordinal, result.name)) = result;
   // track added recipes because we may need to undo them in tests; see below
+//?   LOG << "recently added recipe: " << result.name << ' ' << get(Recipe_ordinal, result.name) << '\n';
   Recently_added_recipes.push_back(get(Recipe_ordinal, result.name));
   return get(Recipe_ordinal, result.name);
 }
@@ -237,16 +238,20 @@ void show_rest_of_stream(istream& in) {
 vector<recipe_ordinal> Recently_added_recipes;
 long long int Reserved_for_tests = 1000;
 :(before "End Setup")
-for (long long int i = 0; i < SIZE(Recently_added_recipes); ++i) {
-  if (Recently_added_recipes.at(i) >= Reserved_for_tests  // don't renumber existing recipes, like 'interactive'
-      && contains_key(Recipe, Recently_added_recipes.at(i)))  // in case previous test had duplicate definitions
-    Recipe_ordinal.erase(get(Recipe, Recently_added_recipes.at(i)).name);
-  Recipe.erase(Recently_added_recipes.at(i));
+clear_recently_added_recipes();
+:(code)
+void clear_recently_added_recipes() {
+  for (long long int i = 0; i < SIZE(Recently_added_recipes); ++i) {
+    if (Recently_added_recipes.at(i) >= Reserved_for_tests  // don't renumber existing recipes, like 'interactive'
+        && contains_key(Recipe, Recently_added_recipes.at(i)))  // in case previous test had duplicate definitions
+      Recipe_ordinal.erase(get(Recipe, Recently_added_recipes.at(i)).name);
+//?   LOG << "erase recipe " << Recently_added_recipes.at(i) << ' ' << get(Recipe, Recently_added_recipes.at(i)).name << '\n';
+    Recipe.erase(Recently_added_recipes.at(i));
+  }
+  // Clear Other State For Recently_added_recipes
+  Recently_added_recipes.clear();
 }
-// Clear Other State For Recently_added_recipes
-Recently_added_recipes.clear();
 
-:(code)
 :(scenario parse_comment_outside_recipe)
 # this comment will be dropped by the tangler, so we need a dummy recipe to stop that
 recipe f1 [ ]
diff --git a/029tools.cc b/029tools.cc
index 29b40836..fb52c7b4 100644
--- a/029tools.cc
+++ b/029tools.cc
@@ -301,3 +301,29 @@ case _DUMP_MEMORY: {
   dump_memory();
   break;
 }
+
+//: In times of real extremis we need to create a whole new modality for debug
+//: logs, independent of other changes to the screen or Trace_stream.
+
+:(before "End Globals")
+ofstream LOG;
+:(before "End One-time Setup")
+//? LOG.open("log");
+
+:(before "End Primitive Recipe Declarations")
+_LOG,
+:(before "End Primitive Recipe Numbers")
+put(Recipe_ordinal, "$log", _LOG);
+:(before "End Primitive Recipe Checks")
+case _LOG: {
+  break;
+}
+:(before "End Primitive Recipe Implementations")
+case _LOG: {
+  ostringstream out;
+  for (long long int i = 0; i < SIZE(current_instruction().ingredients); ++i) {
+    out << print_mu(current_instruction().ingredients.at(i), ingredients.at(i));
+  }
+  LOG << out.str() << "(length: " << get(Recipe_ordinal, "length") << '/' << contains_key(Recipe, get(Recipe_ordinal, "length")) << ")\n";
+  break;
+}
diff --git a/057static_dispatch.cc b/057static_dispatch.cc
index 60221453..222bd97f 100644
--- a/057static_dispatch.cc
+++ b/057static_dispatch.cc
@@ -21,10 +21,10 @@ recipe test a:number, b:number -> z:number [
 map<string, vector<recipe_ordinal> > Recipe_variants;
 :(before "End One-time Setup")
 put(Recipe_variants, "main", vector<recipe_ordinal>());  // since we manually added main to Recipe_ordinal
-:(before "End Setup")
+:(before "Clear Other State For Recently_added_recipes")
 for (map<string, vector<recipe_ordinal> >::iterator p = Recipe_variants.begin(); p != Recipe_variants.end(); ++p) {
   for (long long int i = 0; i < SIZE(p->second); ++i) {
-    if (p->second.at(i) >= Reserved_for_tests)
+    if (find(Recently_added_recipes.begin(), Recently_added_recipes.end(), p->second.at(i)) != Recently_added_recipes.end())
       p->second.at(i) = -1;  // just leave a ghost
   }
 }
@@ -32,30 +32,41 @@ for (map<string, vector<recipe_ordinal> >::iterator p = Recipe_variants.begin();
 :(before "End Load Recipe Header(result)")
 if (contains_key(Recipe_ordinal, result.name)) {
   const recipe_ordinal r = get(Recipe_ordinal, result.name);
-  if ((!contains_key(Recipe, r) || get(Recipe, r).has_header)
-      && !variant_already_exists(result)) {
-    string new_name = next_unused_recipe_name(result.name);
-    put(Recipe_ordinal, new_name, Next_recipe_ordinal++);
-    get_or_insert(Recipe_variants, result.name).push_back(get(Recipe_ordinal, new_name));
+//?   LOG << "checking " << r << " " << result.name << '\n';
+//?   cerr << result.name << ": " << contains_key(Recipe, r) << (contains_key(Recipe, r) ? get(Recipe, r).has_header : 0) << matching_variant_name(result) << '\n';
+  if (!contains_key(Recipe, r) || get(Recipe, r).has_header) {
+    string new_name = matching_variant_name(result);
+    if (new_name.empty()) {
+      // variant doesn't already exist
+      new_name = next_unused_recipe_name(result.name);
+//?       LOG << "adding a variant of " << result.name << ": " << new_name << " is now " << Next_recipe_ordinal << '\n';
+      put(Recipe_ordinal, new_name, Next_recipe_ordinal++);
+      get_or_insert(Recipe_variants, result.name).push_back(get(Recipe_ordinal, new_name));
+    }
     result.name = new_name;
+//?     cerr << "=> " << new_name << '\n';
   }
 }
 else {
   // save first variant
+//?   LOG << "saving first variant of " << result.name << ": " << Next_recipe_ordinal << '\n';
   put(Recipe_ordinal, result.name, Next_recipe_ordinal++);
   get_or_insert(Recipe_variants, result.name).push_back(get(Recipe_ordinal, result.name));
 }
 
 :(code)
-bool variant_already_exists(const recipe& rr) {
+string matching_variant_name(const recipe& rr) {
   const vector<recipe_ordinal>& variants = get_or_insert(Recipe_variants, rr.name);
   for (long long int i = 0; i < SIZE(variants); ++i) {
-    if (contains_key(Recipe, variants.at(i))
-        && all_reagents_match(rr, get(Recipe, variants.at(i)))) {
-      return true;
-    }
+//?     LOG << "checking variant " << variants.at(i) << " of " << rr.name << '\n';
+    if (!contains_key(Recipe, variants.at(i))) continue;
+    const recipe& candidate = get(Recipe, variants.at(i));
+    if (!all_reagents_match(rr, candidate)) continue;
+//?     LOG << "  exists\n";
+    return candidate.name;
   }
-  return false;
+//?   LOG << "  does not exist\n";
+  return "";
 }
 
 bool all_reagents_match(const recipe& r1, const recipe& r2) {
@@ -457,3 +468,27 @@ string header_label(recipe_ordinal r) {
     out << ' ' << caller.products.at(i).original_string;
   return out.str();
 }
+
+:(scenario reload_variant_retains_other_variants)
+recipe main [
+  1:number <- copy 34
+  2:number <- foo 1:number
+]
+recipe foo x:number -> y:number [
+  local-scope
+  load-ingredients
+  reply 34
+]
+recipe foo x:address:number -> y:number [
+  local-scope
+  load-ingredients
+  reply 35
+]
+recipe! foo x:address:number -> y:number [
+  local-scope
+  load-ingredients
+  reply 36
+]
++mem: storing 34 in location 2
+$error: 0
+$warn: 0
diff --git a/059shape_shifting_recipe.cc b/059shape_shifting_recipe.cc
index 04c72af8..38abf76f 100644
--- a/059shape_shifting_recipe.cc
+++ b/059shape_shifting_recipe.cc
@@ -74,21 +74,21 @@ if (best_score == -1) {
   if (exemplar) {
 //?     cerr << "specializing " << inst.name << '\n';
     trace(9992, "transform") << "found variant to specialize: " << exemplar << ' ' << get(Recipe, exemplar).name << end();
-//?     cerr << "found variant to specialize: " << exemplar << ' ' << get(Recipe, exemplar).name << '\n';
+    LOG << "found variant to specialize: " << exemplar << ' ' << header(get(Recipe, exemplar)) << '\n';
     recipe_ordinal new_recipe_ordinal = new_variant(exemplar, inst, caller_recipe);
     variants.push_back(new_recipe_ordinal);
     // perform all transforms on the new specialization
     const string& new_name = get(Recipe, variants.back()).name;
     trace(9992, "transform") << "transforming new specialization: " << new_name << end();
-//?     cerr << "transforming new specialization: " << new_name << '\n';
+    LOG << "transforming new specialization: " << header(get(Recipe, variants.back())) << '\n';
     for (long long int t = 0; t < SIZE(Transform); ++t) {
       (*Transform.at(t))(new_recipe_ordinal);
     }
     get(Recipe, new_recipe_ordinal).transformed_until = SIZE(Transform)-1;
-//?     cerr << "-- replacing " << inst.name << " with " << get(Recipe, variants.back()).name << '\n' << debug_string(get(Recipe, variants.back()));
+    LOG << "replacing " << inst.name << " with " << get(Recipe, variants.back()).name << '\n';
     inst.name = get(Recipe, variants.back()).name;
     trace(9992, "transform") << "new specialization: " << inst.name << end();
-//?     cerr << "new specialization: " << inst.name << '\n';
+    LOG << "new specialization: " << inst.name << '\n';
   }
 }
 
@@ -96,13 +96,34 @@ if (best_score == -1) {
 //: before running mu programs
 
 :(before "End Instruction Operation Checks")
-if (contains_key(Recipe, inst.operation)
+//? LOG << inst.operation << " " << contains_key(Recipe, inst.operation) << '\n';
+if (contains_key(Recipe, inst.operation) && inst.operation >= MAX_PRIMITIVE_RECIPES
     && any_type_ingredient_in_header(inst.operation)) {
+//?   LOG << header(caller) << "instruction " << inst.name << " has no valid specialization\n";
   raise_error << maybe(caller.name) << "instruction " << inst.name << " has no valid specialization\n" << end();
   return;
 }
 
 :(code)
+string header(const recipe& caller) {
+  if (!caller.has_header) return maybe(caller.name);
+  ostringstream out;
+  out << caller.name;
+  for (long long int i = 0; i < SIZE(caller.ingredients); ++i) {
+    if (i > 0) out << ',';
+    out << ' ' << debug_string(caller.ingredients.at(i));
+  }
+  if (!caller.products.empty()) {
+    out << " ->";
+    for (long long int i = 0; i < SIZE(caller.products); ++i) {
+      if (i > 0) out << ',';
+      out << ' ' << debug_string(caller.products.at(i));
+    }
+  }
+  out << ": ";
+  return out.str();
+}
+
 recipe_ordinal pick_matching_shape_shifting_variant(vector<recipe_ordinal>& variants, const instruction& inst, long long int& best_score) {
 //?   cerr << "---- " << inst.name << ": " << non_ghost_size(variants) << '\n';
   recipe_ordinal result = 0;
diff --git a/080display.cc b/080display.cc
index 970887bd..6f92a0f6 100644
--- a/080display.cc
+++ b/080display.cc
@@ -412,7 +412,11 @@ case CHECK_FOR_INTERACTION: {
   // treat keys within ascii as unicode characters
   if (event_type == TB_EVENT_KEY && event.key < 0xff) {
     products.at(0).push_back(/*text event*/0);
-    if (event.key == TB_KEY_CTRL_C) tb_shutdown(), exit(1);
+    if (event.key == TB_KEY_CTRL_C) {
+      tb_shutdown();
+//?       LOG << "exit\n";
+      exit(1);
+    }
     if (event.key == TB_KEY_BACKSPACE2) event.key = TB_KEY_BACKSPACE;
     if (event.key == TB_KEY_CARRIAGE_RETURN) event.key = TB_KEY_NEWLINE;
     products.at(0).push_back(event.key);
diff --git a/091run_interactive.cc b/091run_interactive.cc
index 0396ad47..28c87994 100644
--- a/091run_interactive.cc
+++ b/091run_interactive.cc
@@ -62,6 +62,8 @@ bool Track_most_recent_products = false;
 :(before "End Tracing")
 trace_stream* Save_trace_stream = NULL;
 string Save_trace_file;
+vector<recipe_ordinal> Save_recently_added_recipes;
+vector<recipe_ordinal> Save_recently_added_shape_shifting_recipes;
 :(before "End Setup")
 Track_most_recent_products = false;
 :(code)
@@ -79,7 +81,7 @@ bool run_interactive(long long int address) {
   string command = trim(strip_comments(read_mu_string(address)));
   if (command.empty()) return false;
   Name[get(Recipe_ordinal, "interactive")].clear();
-  run_code_begin();
+  run_code_begin(/*snapshot_recently_added_recipes*/true);
   // don't kill the current routine on parse errors
   routine* save_current_routine = Current_routine;
   Current_routine = NULL;
@@ -106,12 +108,18 @@ bool run_interactive(long long int address) {
   return true;
 }
 
-void run_code_begin() {
+void run_code_begin(bool snapshot_recently_added_recipes) {
 //?   cerr << "loading new trace\n";
   // stuff to undo later, in run_code_end()
   Hide_warnings = true;
   Hide_errors = true;
   Disable_redefine_warnings = true;
+  if (snapshot_recently_added_recipes) {
+    Save_recently_added_recipes = Recently_added_recipes;
+    Recently_added_recipes.clear();
+    Save_recently_added_shape_shifting_recipes = Recently_added_shape_shifting_recipes;
+    Recently_added_shape_shifting_recipes.clear();
+  }
   Save_trace_stream = Trace_stream;
   Save_trace_file = Trace_file;
   Trace_file = "";
@@ -130,6 +138,13 @@ void run_code_end() {
   Trace_file = Save_trace_file;
   Save_trace_file.clear();
   Recipe.erase(get(Recipe_ordinal, "interactive"));  // keep past sandboxes from inserting errors
+  if (!Save_recently_added_recipes.empty()) {
+    clear_recently_added_recipes();
+    Recently_added_recipes = Save_recently_added_recipes;
+    Save_recently_added_recipes.clear();
+    Recently_added_shape_shifting_recipes = Save_recently_added_shape_shifting_recipes;
+    Save_recently_added_shape_shifting_recipes.clear();
+  }
 }
 
 :(before "End Load Recipes")
@@ -304,6 +319,31 @@ b:number <- copy 0
 # no errors
 +mem: storing 0 in location 3
 
+:(code)
+void test_run_interactive_cleans_up_any_created_specializations() {
+  // define a generic recipe
+  assert(!contains_key(Recipe_ordinal, "foo"));
+  load("recipe foo x:_elem -> n:number [\n"
+       "  reply 34\n"
+       "]\n");
+  assert(SIZE(Recently_added_recipes) == 1);  // foo
+  assert(variant_count("foo") == 1);
+  // run-interactive a call that specializes this recipe
+  run("recipe main [\n"
+       "  1:number/raw <- copy 0\n"
+       "  2:address:array:character <- new [foo 1:number/raw]\n"
+       "  run-interactive 2:address:array:character\n"
+       "]\n");
+  assert(SIZE(Recently_added_recipes) == 2);  // foo, main
+  // check that number of variants doesn't change
+  CHECK_EQ(variant_count("foo"), 1);
+}
+
+long long int variant_count(string recipe_name) {
+  if (!contains_key(Recipe_variants, recipe_name)) return 0;
+  return non_ghost_size(get(Recipe_variants, recipe_name));
+}
+
 :(before "End Globals")
 string Most_recent_products;
 :(before "End Setup")
@@ -438,13 +478,13 @@ case RELOAD: {
     }
   }
   for (long long int i = 0; i < SIZE(Recently_added_shape_shifting_recipes); ++i) {
-//?     cerr << "erasing " << get(Recipe, Recently_added_shape_shifting_recipes.at(i)).name << '\n';
+//?     LOG << "erasing " << get(Recipe, Recently_added_shape_shifting_recipes.at(i)).name << '\n';
     Recipe_ordinal.erase(get(Recipe, Recently_added_shape_shifting_recipes.at(i)).name);
     Recipe.erase(Recently_added_shape_shifting_recipes.at(i));
   }
   Recently_added_shape_shifting_recipes.clear();
   string code = read_mu_string(ingredients.at(0).at(0));
-  run_code_begin();
+  run_code_begin(/*snapshot_recently_added_recipes*/false);
   routine* save_current_routine = Current_routine;
   Current_routine = NULL;
   vector<recipe_ordinal> recipes_reloaded = load(code);
@@ -472,3 +512,29 @@ recipe main [
   1:number/raw <- copy 34
 ]
 +mem: storing 34 in location 1
+
+:(code)
+void test_reload_cleans_up_any_created_specializations() {
+  // define a generic recipe and a call to it
+  assert(!contains_key(Recipe_ordinal, "foo"));
+  assert(variant_count("foo") == 0);
+  // a call that specializes this recipe
+  run("recipe main [\n"
+      "  local-scope\n"
+      "  x:address:array:character <- new [recipe foo x:_elem -> n:number [\n"
+      "local-scope\n"
+      "load-ingredients\n"
+      "reply 34\n"
+      "]\n"
+      "recipe main2 [\n"
+      "local-scope\n"
+      "load-ingredients\n"
+      "x:number <- copy 34\n"
+      "foo x:number\n"
+      "]]\n"
+      "  reload x\n"
+      "]\n");
+  // check that number of variants includes specialization
+  assert(SIZE(Recently_added_recipes) == 4);  // foo, main, main2, foo specialization
+  CHECK_EQ(variant_count("foo"), 2);
+}
diff --git a/edit/005-sandbox.mu b/edit/005-sandbox.mu
index 1c024b2b..a8a4c5a3 100644
--- a/edit/005-sandbox.mu
+++ b/edit/005-sandbox.mu
@@ -114,6 +114,7 @@ after <global-keypress> [
   {
     do-run?:boolean <- equal *k, 65532/F4
     break-unless do-run?
+#?     $log [F4 pressed]
     status:address:array:character <- new [running...  ]
     screen <- update-status screen, status, 245/grey
     error?:boolean, env, screen <- run-sandboxes env, screen
@@ -226,6 +227,7 @@ recipe save-sandboxes env:address:programming-environment-data [
 recipe! render-sandbox-side screen:address:screen, env:address:programming-environment-data -> screen:address:screen [
   local-scope
   load-ingredients
+#?   $log [render sandbox side]
   trace 11, [app], [render sandbox side]
   current-sandbox:address:editor-data <- get *env, current-sandbox:offset
   left:number <- get *current-sandbox, left:offset
@@ -242,6 +244,7 @@ recipe! render-sandbox-side screen:address:screen, env:address:programming-envir
 recipe render-sandboxes screen:address:screen, sandbox:address:sandbox-data, left:number, right:number, row:number -> row:number, screen:address:screen, sandbox:address:sandbox-data [
   local-scope
   load-ingredients
+#?   $log [render sandbox]
   reply-unless sandbox
   screen-height:number <- screen-height screen
   at-bottom?:boolean <- greater-or-equal row, screen-height
diff --git a/edit/010-warnings.mu b/edit/010-warnings.mu
index 5e41cf12..419d9c8f 100644
--- a/edit/010-warnings.mu
+++ b/edit/010-warnings.mu
@@ -8,6 +8,7 @@ container programming-environment-data [
 recipe! update-recipes env:address:programming-environment-data, screen:address:screen -> errors-found?:boolean, env:address:programming-environment-data, screen:address:screen [
   local-scope
   load-ingredients
+#?   $log [update recipes]
   recipes:address:editor-data <- get *env, recipes:offset
   in:address:array:character <- editor-contents recipes
   save [recipes.mu], in
@@ -49,6 +50,7 @@ container sandbox-data [
 recipe! update-sandbox sandbox:address:sandbox-data -> sandbox:address:sandbox-data [
   local-scope
   load-ingredients
+#?   $log [update sandbox]
   data:address:array:character <- get *sandbox, data:offset
   response:address:address:array:character <- get-address *sandbox, response:offset
   warnings:address:address:array:character <- get-address *sandbox, warnings:offset
@@ -199,6 +201,66 @@ z <- add x, [a]
   ]
 ]
 
+scenario run-avoids-spurious-warnings-on-reloading-shape-shifting-recipes [
+  trace-until 100/app  # trace too long
+  assume-screen 100/width, 15/height
+  # overload a well-known shape-shifting recipe
+  1:address:array:character <- new [recipe length l:address:list:_elem -> n:number [
+]]
+  # call code that uses other variants of it, but not it itself
+  2:address:array:character <- new [x:address:list:number <- copy 0
+to-text x]
+  3:address:programming-environment-data <- new-programming-environment screen:address:screen, 1:address:array:character, 2:address:array:character
+  # run it once
+  assume-console [
+    press F4
+  ]
+  event-loop screen:address:screen, console:address:console, 3:address:programming-environment-data
+  # no errors anywhere on screen (can't check anything else, since to-text will return an address)
+  screen-should-contain-in-color 1/red, [
+    .                                                                                                    .
+    .                                                                                                    .
+    .                                                                                                    .
+    .                                                                                                    .
+    .                                                                         <-                         .
+    .                                                                                                    .
+    .                                                                                                    .
+    .                                                                                                    .
+    .                                                                                                    .
+    .                                                                                                    .
+    .                                                                                                    .
+    .                                                                                                    .
+    .                                                                                                    .
+    .                                                                                                    .
+    .                                                                                                    .
+  ]
+  # rerun everything
+  assume-console [
+    press F4
+  ]
+  run [
+    event-loop screen:address:screen, console:address:console, 3:address:programming-environment-data
+  ]
+  # still no errors
+  screen-should-contain-in-color 1/red, [
+    .                                                                                                    .
+    .                                                                                                    .
+    .                                                                                                    .
+    .                                                                                                    .
+    .                                                                         <-                         .
+    .                                                                                                    .
+    .                                                                                                    .
+    .                                                                                                    .
+    .                                                                                                    .
+    .                                                                                                    .
+    .                                                                                                    .
+    .                                                                                                    .
+    .                                                                                                    .
+    .                                                                                                    .
+    .                                                                                                    .
+  ]
+]
+
 scenario run-shows-missing-type-warnings [
   trace-until 100/app  # trace too long
   assume-screen 100/width, 15/height