2393 - redo 2391

Got that idea to work with a special-case for 'new'. Requires parsing
new's first ingredient, performing the replacement, and then turning it
back into a string. I didn't want to replace NEW with ALLOCATE right
here, because then it messes with my invariant that transform should
never see a naked ALLOCATE.

Layer 11 still not working, but everything else is. Let's clean up
before we diagnose the new breakage.

9 files changed, 166 insertions, 61 deletions

diff --git a/001help.cc b/001help.cc
index e42dc6a7..c193cf7d 100644
--- a/001help.cc
+++ b/001help.cc
@@ -96,6 +96,11 @@ template<typename T> typename T::mapped_type& get(T& map, typename T::key_type c
   assert(iter != map.end());
   return iter->second;
 }
+template<typename T> typename T::mapped_type const& get(const T& map, typename T::key_type const& key) {
+  typename T::const_iterator iter(map.find(key));
+  assert(iter != map.end());
+  return iter->second;
+}
 template<typename T> typename T::mapped_type const& put(T& map, typename T::key_type const& key, typename T::mapped_type const& value) {
   map[key] = value;
   return map[key];
diff --git a/010vm.cc b/010vm.cc
index a4c5eb54..9ac47f6f 100644
--- a/010vm.cc
+++ b/010vm.cc
@@ -278,8 +278,10 @@ type_tree* new_type_tree(const string_tree* properties) {
       result->value = get(Type_ordinal, type_name);
     else if (is_integer(type_name))  // sometimes types will contain non-type tags, like numbers for the size of an array
       result->value = 0;
-    else
+    else {
+      cerr << "AAAAAAAAAAAAAA "; dump_property(properties, cerr); cerr << '\n';
       result->value = -1;  // should never happen; will trigger errors later
+    }
   }
   result->left = new_type_tree(properties->left);
   result->right = new_type_tree(properties->right);
diff --git a/012transform.cc b/012transform.cc
index b5266ed6..7837ae67 100644
--- a/012transform.cc
+++ b/012transform.cc
@@ -37,7 +37,7 @@ void transform_all() {
       recipe& r = p->second;
       if (r.steps.empty()) continue;
       if (r.transformed_until != t-1) continue;
-//?       cerr << "  recipe " << r.name << '\n';
+      // End Transform Checks
       (*Transform.at(t))(/*recipe_ordinal*/p->first);
       r.transformed_until = t;
     }
diff --git a/031address.cc b/031address.cc
index c4b260d1..79c951d0 100644
--- a/031address.cc
+++ b/031address.cc
@@ -78,7 +78,8 @@ void lookup_memory(reagent& x) {
 bool canonize_type(reagent& r) {
   while (has_property(r, "lookup")) {
     if (!r.type || r.type->value != get(Type_ordinal, "address")) {
-      raise_error << "can't lookup non-address: " << r.original_string << '\n' << end();
+      raise_error << "can't lookup non-address: " << r.to_string() << '\n' << end();
+      dump_types(r.type, cerr);  cerr << '\n';
       return false;
     }
     drop_address_from_type(r);
diff --git a/043new.cc b/043new.cc
index bc552362..96b2fe37 100644
--- a/043new.cc
+++ b/043new.cc
@@ -61,14 +61,11 @@ void transform_new_to_allocate(const recipe_ordinal r) {
     // Convert 'new' To 'allocate'
     if (inst.name == "new") {
       inst.operation = ALLOCATE;
-//?       istringstream in(inst.ingredients.at(0).name);
-//?       in >> std::noskipws;
-      cerr << "new: " << inst.ingredients.at(0).name << '\n';
+      cerr << inst.ingredients.at(0).name << '\n';
       string_tree* type_name = new string_tree(inst.ingredients.at(0).name);
       // End Post-processing(type_name) When Converting 'new'
       type_tree* type = new_type_tree(type_name);
       inst.ingredients.at(0).set_value(size_of(type));
-      cerr << "=> " << inst.ingredients.at(0).value << '\n';
       ostringstream out;
       dump_property(type_name, out);
       trace(9992, "new") << "size of " << out.str() << " is " << inst.ingredients.at(0).value << end();
diff --git a/048check_type_by_name.cc b/048check_type_by_name.cc
index 8a83c5f4..27807103 100644
--- a/048check_type_by_name.cc
+++ b/048check_type_by_name.cc
@@ -36,6 +36,15 @@ void check_types_by_name(const recipe_ordinal r) {
   }
 }
 
+void deduce_missing_type(map<string, type_tree*>& type, map<string, string_tree*>& type_name, reagent& x) {
+  if (x.type) return;
+  if (!contains_key(type, x.name)) return;
+  x.type = new type_tree(*type[x.name]);
+  trace(9992, "transform") << x.name << " <= " << dump_types(x) << end();
+  assert(!x.properties.at(0).second);
+  x.properties.at(0).second = new string_tree(*type_name[x.name]);
+}
+
 void check_type(map<string, type_tree*>& type, map<string, string_tree*>& type_name, const reagent& x, const recipe_ordinal r) {
   if (is_literal(x)) return;
   if (is_raw(x)) return;  // TODO: delete this
@@ -59,16 +68,6 @@ recipe main [
   y:number <- add x, 1
 ]
 
-:(code)
-void deduce_missing_type(map<string, type_tree*>& type, map<string, string_tree*>& type_name, reagent& x) {
-  if (x.type) return;
-  if (!contains_key(type, x.name)) return;
-  x.type = new type_tree(*type[x.name]);
-  trace(9992, "transform") << x.name << " <= " << dump_types(x) << end();
-  assert(!x.properties.at(0).second);
-  x.properties.at(0).second = new string_tree(*type_name[x.name]);
-}
-
 :(scenario transform_fills_in_missing_types_in_product)
 recipe main [
   x:number <- copy 1
diff --git a/057static_dispatch.cc b/057static_dispatch.cc
index b8b663e2..a52cda9a 100644
--- a/057static_dispatch.cc
+++ b/057static_dispatch.cc
@@ -112,7 +112,7 @@ Transform.push_back(resolve_ambiguous_calls);  // idempotent
 void resolve_ambiguous_calls(recipe_ordinal r) {
   if (!get(Recipe, r).has_header) return;
   trace(9991, "transform") << "--- resolve ambiguous calls for recipe " << get(Recipe, r).name << end();
-//?   cerr << "--- resolve ambiguous calls for recipe " << get(Recipe, r).name << '\n';
+  cerr << "--- resolve ambiguous calls for recipe " << get(Recipe, r).name << '\n';
   for (long long int index = 0; index < SIZE(get(Recipe, r).steps); ++index) {
     instruction& inst = get(Recipe, r).steps.at(index);
     if (inst.is_label) continue;
diff --git a/058generic_container.cc b/058generic_container.cc
index 44762f00..df9b63bc 100644
--- a/058generic_container.cc
+++ b/058generic_container.cc
@@ -60,15 +60,15 @@ void read_type_ingredients(string& name) {
       raise_error << "can't repeat type ingredient names in a single container definition\n" << end();
       return;
     }
-    info.type_ingredient_names[curr] = next_type_ordinal++;
+    put(info.type_ingredient_names, curr, next_type_ordinal++);
   }
 }
 
 :(before "End insert_container Special Uses(type_name)")
 // check for use of type ingredients
 if (type_name.at(0) == '_') {
-  *curr_type = new type_tree(info.type_ingredient_names[type_name]);
-  trace(9999, "parse") << "  type: " << info.type_ingredient_names[type_name] << end();
+  *curr_type = new type_tree(get(info.type_ingredient_names, type_name));
+  trace(9999, "parse") << "  type: " << get(info.type_ingredient_names, type_name) << end();
   continue;
 }
 
@@ -86,7 +86,7 @@ if (t.elements.at(i)->value >= START_TYPE_INGREDIENTS) {
     dump_types(type, out);
     raise_error << "illegal type '" << out.str() << "' seems to be missing a type ingredient or three\n" << end();
   }
-  result += size_of_type_ingredient(t.elements.at(i), type->right);
+  result += size;
   continue;
 }
 
@@ -103,6 +103,7 @@ long long int size_of_type_ingredient(const type_tree* element_template, const t
   }
   assert(curr);
   assert(!curr->left);  // unimplemented
+  assert(contains_key(Type, curr->value));
   trace(9999, "type") << "type deduced to be " << get(Type, curr->value).name << "$" << end();
   type_tree tmp(curr->value);
   if (curr->right)
diff --git a/059generic_recipe.cc b/059generic_recipe.cc
index 64dbf6a1..3588f183 100644
--- a/059generic_recipe.cc
+++ b/059generic_recipe.cc
@@ -20,19 +20,10 @@ recipe foo a:_t -> result:_t [
 +mem: storing 14 in location 11
 +mem: storing 15 in location 12
 
-//: Before anything else, disable all previous transforms which rely on
-//: reagent.type if a recipe contains any type ingredients.
+//: Before anything else, disable transforms for generic recipes.
 
-:(after "void check_instruction(const recipe_ordinal r)")
-  if (any_type_ingredient_in_header(r)) return;
-:(after "void check_or_set_invalid_types(const recipe_ordinal r)")
-  if (any_type_ingredient_in_header(r)) return;
-:(after "void transform_names(const recipe_ordinal r)")
-  if (any_type_ingredient_in_header(r)) return;
-:(after "void transform_new_to_allocate(const recipe_ordinal r)")
-  if (any_type_ingredient_in_header(r)) return;
-:(after "void check_header_products(const recipe_ordinal r)")
-  if (any_type_ingredient_in_header(r)) return;
+:(before "End Transform Checks")
+if (any_type_ingredient_in_header(/*recipe_ordinal*/p->first)) continue;
 
 //: We'll be creating recipes without loading them from anywhere by
 //: *specializing* existing recipes, so make sure we don't clear any of those
@@ -50,16 +41,9 @@ if (best_score == -1) {
     variants.push_back(new_variant(exemplar, inst));
     inst.name = get(Recipe, variants.back()).name;
     trace(9992, "transform") << "new specialization: " << inst.name << end();
-//?     cerr << "new specialization: " << inst.name << '\n';
   }
 }
 
-//: Don't bother resolving ambiguous calls inside generic recipes. Just do
-//: their specializations.
-
-:(after "void resolve_ambiguous_calls")
-if (any_type_ingredient_in_header(r)) return;
-
 :(code)
 recipe_ordinal pick_matching_generic_variant(vector<recipe_ordinal>& variants, const instruction& inst, long long int& best_score) {
   recipe_ordinal result = 0;
@@ -140,42 +124,95 @@ recipe_ordinal new_variant(recipe_ordinal exemplar, const instruction& inst) {
   string new_name = next_unused_recipe_name(inst.name);
   trace(9993, "transform") << "switching " << inst.name << " to " << new_name << end();
   assert(!contains_key(Recipe_ordinal, new_name));
-  recipe_ordinal result = put(Recipe_ordinal, new_name, Next_recipe_ordinal++);
+  recipe_ordinal new_recipe_ordinal = put(Recipe_ordinal, new_name, Next_recipe_ordinal++);
   // make a copy
   assert(contains_key(Recipe, exemplar));
-  assert(!contains_key(Recipe, result));
-  recently_added_recipes.push_back(result);
-  put(Recipe, result, get(Recipe, exemplar));
-  recipe& new_recipe = get(Recipe, result);
-  // update its name
+  assert(!contains_key(Recipe, new_recipe_ordinal));
+  recently_added_recipes.push_back(new_recipe_ordinal);
+  put(Recipe, new_recipe_ordinal, get(Recipe, exemplar));
+  recipe& new_recipe = get(Recipe, new_recipe_ordinal);
+  // Since the exemplar never ran any transforms, we have to redo some of the
+  // work of the check_types_by_name transform while supporting type-ingredients.
+  compute_type_names(new_recipe);
+  // that gives enough information to replace type-ingredients with concrete types
   new_recipe.name = new_name;
-  // update its contents
-  map<string, string> mappings;  // weak references
+  map<string, string> mappings;
   compute_type_ingredient_mappings(get(Recipe, exemplar), inst, mappings);
   replace_type_ingredients(new_recipe, mappings);
-  return result;
+  ensure_all_concrete_types(new_recipe);
+  // finally, perform all transforms on the new specialization
+  cerr << "contents of " << new_recipe.name << '\n';
+  for (long long int index = 0; index < SIZE(new_recipe.steps); ++index) {
+    instruction& inst = new_recipe.steps.at(index);
+    cerr << "inst: " << inst.to_string() << '\n';
+    for (long long int i = 0; i < SIZE(inst.ingredients); ++i) {
+      cerr << "  " << inst.ingredients.at(i).to_string() << " => " << dump_types(inst.ingredients.at(i)) << '\n';
+    }
+    cerr << "--\n";
+    for (long long int i = 0; i < SIZE(inst.products); ++i)
+      cerr << "  " << inst.products.at(i).to_string() << " => " << dump_types(inst.products.at(i)) << '\n';
+  }
+  for (long long int t = 0; t < SIZE(Transform); ++t) {
+    (*Transform.at(t))(new_recipe_ordinal);
+  }
+  new_recipe.transformed_until = SIZE(Transform)-1;
+  return new_recipe_ordinal;
+}
+
+void compute_type_names(recipe& variant) {
+  map<string, string_tree*> type_names;
+  for (long long int i = 0; i < SIZE(variant.ingredients); ++i) {
+    save_or_deduce_type_name(variant.ingredients.at(i), type_names);
+  }
+  for (long long int i = 0; i < SIZE(variant.products); ++i) {
+    save_or_deduce_type_name(variant.products.at(i), type_names);
+  }
+  for (long long int i = 0; i < SIZE(variant.steps); ++i) {
+    instruction& inst = variant.steps.at(i);
+    for (long long int in = 0; in < SIZE(inst.ingredients); ++in) {
+      save_or_deduce_type_name(inst.ingredients.at(in), type_names);
+    }
+    for (long long int out = 0; out < SIZE(inst.products); ++out) {
+      save_or_deduce_type_name(inst.products.at(out), type_names);
+    }
+  }
+}
+
+void save_or_deduce_type_name(reagent& x, map<string, string_tree*>& type_name) {
+  if (!x.properties.at(0).second && contains_key(type_name, x.name)) {
+    x.properties.at(0).second = new string_tree(*get(type_name, x.name));
+    return;
+  }
+  if (!x.properties.at(0).second) {
+    raise << "unknown type for " << x.original_string << '\n' << end();
+    return;
+  }
+  if (contains_key(type_name, x.name)) return;
+  if (x.properties.at(0).second->value == "offset" || x.properties.at(0).second->value == "variant") return;  // special-case for container-access instructions
+  put(type_name, x.name, x.properties.at(0).second);
+  ostringstream type_name_buf;
+  dump_property(x.properties.at(0).second, type_name_buf);
+  trace(9993, "transform") << "type of " << x.name << " is " << type_name_buf.str() << end();
 }
 
 void compute_type_ingredient_mappings(const recipe& exemplar, const instruction& inst, map<string, string>& mappings) {
-//?   cerr << "=== " << exemplar.name << " vs " << inst.to_string() << '\n';
   for (long long int i = 0; i < SIZE(exemplar.ingredients); ++i) {
+    const reagent& base = exemplar.ingredients.at(i);
     reagent ingredient = inst.ingredients.at(i);
+    assert(ingredient.properties.at(0).second);
     canonize_type(ingredient);
-    accumulate_type_ingredients(exemplar.ingredients.at(i), ingredient, mappings, exemplar);
+    accumulate_type_ingredients(base, ingredient, mappings, exemplar);
   }
   for (long long int i = 0; i < SIZE(exemplar.products); ++i) {
+    const reagent& base = exemplar.products.at(i);
     reagent product = inst.products.at(i);
+    assert(product.properties.at(0).second);
     canonize_type(product);
-    accumulate_type_ingredients(exemplar.products.at(i), product, mappings, exemplar);
+    accumulate_type_ingredients(base, product, mappings, exemplar);
   }
 }
 
-void accumulate_type_ingredients(const reagent& base, const reagent& refinement, map<string, string>& mappings, const recipe& exemplar) {
-//?   cerr << base.to_string() << " vs " << refinement.to_string() << '\n';
-  if (!refinement.properties.at(0).second) {
-    if (!Trace_stream) cerr << "Turn on START_TRACING_UNTIL_END_OF_SCOPE in 020run.cc for more details.\n";
-    DUMP("");
-  }
+void accumulate_type_ingredients(const reagent& base, reagent& refinement, map<string, string>& mappings, const recipe& exemplar) {
   assert(refinement.properties.at(0).second);
   accumulate_type_ingredients(base.properties.at(0).second, refinement.properties.at(0).second, mappings, exemplar, base);
 }
@@ -190,10 +227,10 @@ void accumulate_type_ingredients(const string_tree* base, const string_tree* ref
     assert(!refinement->value.empty());
     if (!contains_key(mappings, base->value)) {
       trace(9993, "transform") << "adding mapping from " << base->value << " to " << refinement->value << end();
+      cerr << "adding mapping from " << base->value << " to " << refinement->value << '\n';
       put(mappings, base->value, refinement->value);
     }
     else {
-//?       cerr << base->value << ": " << get(mappings, base->value) << " => " << refinement->value << '\n';
       assert(get(mappings, base->value) == refinement->value);
     }
   }
@@ -224,13 +261,46 @@ void replace_type_ingredients(recipe& new_recipe, const map<string, string>& map
     for (long long int j = 0; j < SIZE(inst.products); ++j) {
       replace_type_ingredients(inst.products.at(j), mappings);
     }
+    // special-case for new: replace type ingredient in first ingredient *value*
+    if (inst.name == "new" && inst.ingredients.at(0).name.at(0) != '[') {
+      cerr << "about to translate new ingredient: " << inst.ingredients.at(0).name << '\n';
+      string_tree* type_name = parse_string_tree(inst.ingredients.at(0).name);
+      dump_property(type_name, cerr); cerr << '\n';
+      replace_type_ingredients(type_name, mappings);
+      cerr << "=> "; dump_property(type_name, cerr); cerr << '\n';
+      inst.ingredients.at(0).name = simple_string(type_name);
+      delete type_name;
+    }
+  }
+}
+
+string simple_string(string_tree* x) {
+  ostringstream out;
+  simple_string(x, out);
+  cerr << "translated new ingredient: " << out.str() << '\n';
+  return out.str();
+}
+
+void simple_string(string_tree* x, ostream& out) {
+  if (!x->left && !x->right) {
+    out << x->value;
+    return;
+  }
+  out << '(';
+  for (string_tree* curr = x; curr; curr = curr->right) {
+    if (curr != x) out << ' ';
+    if (curr->left)
+      simple_string(curr->left, out);
+    else
+      out << curr->value;
   }
+  out << ')';
 }
 
 void replace_type_ingredients(reagent& x, const map<string, string>& mappings) {
-  if (!x.type) return;
   trace(9993, "transform") << "replacing in ingredient " << x.original_string << end();
   // replace properties
+  assert(x.properties.at(0).second);
   replace_type_ingredients(x.properties.at(0).second, mappings);
   // refresh types from properties
   delete x.type;
@@ -243,12 +313,42 @@ void replace_type_ingredients(string_tree* type, const map<string, string>& mapp
   if (!type) return;
   if (is_type_ingredient_name(type->value) && contains_key(mappings, type->value)) {
     trace(9993, "transform") << type->value << " => " << mappings.find(type->value)->second << end();
+    cerr << type->value << " => " << mappings.find(type->value)->second << '\n';
     type->value = mappings.find(type->value)->second;
   }
   replace_type_ingredients(type->left, mappings);
   replace_type_ingredients(type->right, mappings);
 }
 
+void ensure_all_concrete_types(const recipe& new_recipe) {
+  for (long long int i = 0; i < SIZE(new_recipe.ingredients); ++i) {
+    ensure_all_concrete_types(new_recipe.ingredients.at(i).type);
+  }
+  for (long long int i = 0; i < SIZE(new_recipe.products); ++i) {
+    ensure_all_concrete_types(new_recipe.products.at(i).type);
+  }
+  for (long long int i = 0; i < SIZE(new_recipe.steps); ++i) {
+    const instruction& inst = new_recipe.steps.at(i);
+    for (long long int j = 0; j < SIZE(inst.ingredients); ++j) {
+      ensure_all_concrete_types(inst.ingredients.at(j).type);
+    }
+    for (long long int j = 0; j < SIZE(inst.products); ++j) {
+      ensure_all_concrete_types(inst.products.at(j).type);
+    }
+  }
+}
+
+void ensure_all_concrete_types(const type_tree* x) {
+  if (!x) {
+    raise << "AAA null type\n" << end();
+    return;
+  }
+  if (x->value == -1) {
+    raise << "AAA unknown type\n" << end();
+    return;
+  }
+}
+
 :(scenario generic_recipe_2)
 recipe main [
   10:point <- merge 14, 15