gollvm: rework circular function type handling

CL 123738 changes the frontend to explicitly resolve circular
types explicitly, so we don't depend on the order of calls of
placeholder_pointer_type, circular_pointer_type,
set_placeholder_pointer_type, and the construction of the actual
concrete type to resolve circular types.

For circular types, we intercept the cycle with the marker type,
so we won't generate a self-referential type, and link the
marker type (*CPT) to the concrete type.

Adjust test to reflect the new way of API calls by the frontend.

TODO: do the same for circular pointer (non-function) type?

Change-Id: I4264c8913861c7aabae704788b1682cee88bad6b
Reviewed-on: https://go-review.googlesource.com/123796
Reviewed-by: Than McIntosh <thanm@google.com>
diff --git a/bridge/go-llvm-materialize.cpp b/bridge/go-llvm-materialize.cpp
index 3a5d9d1..41d2179 100644
--- a/bridge/go-llvm-materialize.cpp
+++ b/bridge/go-llvm-materialize.cpp
@@ -1449,7 +1449,8 @@
   bool dstCircFunc = isCircularFunctionType(dstToType);
   bool srcCircFunc = isCircularFunctionType(srcType);
   bool srcFuncPtr = isPtrToFuncType(srcType);
-  if ((srcPtrToFD || srcFuncPtr || srcCircFunc) && dstCircFunc) {
+  if (((srcPtrToFD || srcFuncPtr || srcCircFunc) && dstCircFunc) ||
+       (srcCircFunc && dstPtrToFD)) {
     std::string tag(namegen("cast"));
     llvm::Value *bitcast = builder->CreateBitCast(srcVal, dstToType, tag);
     return bitcast;
diff --git a/bridge/go-llvm-typemanager.cpp b/bridge/go-llvm-typemanager.cpp
index 8df2337..69bcfd9 100644
--- a/bridge/go-llvm-typemanager.cpp
+++ b/bridge/go-llvm-typemanager.cpp
@@ -676,49 +676,6 @@
   // Install in cache
   anonTypes_.insert(rval);
 
-  // Handling for circular function types. The recipe used by the front
-  // end is something along the lines of
-  //
-  //     ph = backend()->placeholder_pointer_type(...)
-  //     cpt = backend()->circular_pointer_type(ph, true)
-  //     ft = <create function type involving cpt>
-  //     backend()->set_placeholder_pointer_type(ph, cpt)
-  //
-  // In the sequence above, the placeholder type ("ph") gets pushed
-  // onto the stack below during the call to circular_pointer_type(),
-  // then when the function type is created here in this routine, we
-  // pop the stack and retarget the placeholder to the new function.
-  //
-  if (circularFunctionStack_.size()) {
-    Btype *marker = circularFunctionStack_.back();
-    circularFunctionStack_.pop_back();
-    Btype *pht = circularFunctionStack_.back();
-    circularFunctionStack_.pop_back();
-
-    if (traceLevel() > 1) {
-      std::cerr << "\n^ finalizing placeholder pointer type "
-              << ((void*)pht) << " [llvm type "
-                << ((void*)pht->type()) << "]"
-                << " to point to circular function type "
-                       << ((void*) rval) << " [llvm type "
-                       << ((void*)llft) << "]\n";
-    }
-    Btype *pft = pointerType(rval);
-
-    // This placeholder type most likely has been redirected already,
-    // so re-insert it in the placeholder set (this is somewhat
-    // hacky).  The rationale for calling the general-purpose routine
-    // (setPlaceholderPointerType) here instead of just mutating the
-    // target of the pointer type is that there may be unresolved
-    // placeholders elsewhere in the function type (so we want to use
-    // the more general routine so as to get placeholder reference
-    // tracking).
-    placeholders_.insert(pht);
-    setPlaceholderPointerType(pht, pft);
-    placeholders_.insert(marker);
-    setPlaceholderPointerType(marker, pft);
-  }
-
   return rval;
 }
 
@@ -1033,6 +990,32 @@
       circularFunctionTypes_.find(placeholder->type()) == circularFunctionTypes_.end())
     return true;
 
+  // For circular types, intercept the cycle with the marker type, so we
+  // won't generate a self-referential type.
+  auto cpit = circularPointerTypeMap_.find(placeholder);
+  if (cpit != circularPointerTypeMap_.end()) {
+    Btype *cpt = cpit->second;
+    if (to_type != cpt) {
+      // Link the marker type with the concrete type.
+      BPointerType *bpt = cpt->castToBPointerType();
+      BPointerType *ttpt = to_type->castToBPointerType();
+      bpt->setToType(ttpt->toType());
+
+      if (traceLevel() > 1) {
+        std::cerr << "\n^ link circular pointer type "
+                  << ((void*)cpt) << " [llvm type "
+                  << ((void*)cpt->type()) << "]"
+                  << " to concrete type " << ((void*) to_type)
+                  << " [llvm type " << ((void*) to_type->type())
+                  << "]\n";
+        std::cerr << "marker: "; cpt->dump();
+        std::cerr << "redir: "; to_type->dump();
+      }
+
+      return setPlaceholderPointerType(placeholder, cpt);
+    }
+  }
+
   if (traceLevel() > 1) {
     std::cerr << "\n^ placeholder pointer "
               << ((void*)placeholder) << " [llvm type "
@@ -1277,13 +1260,10 @@
     // update them when the appropriate function type is created.
     circularFunctionPlaceholderTypes_.insert(placeholder);
     circularFunctionTypes_.insert(rval->type());
-    circularFunctionStack_.push_back(placeholder);
-    circularFunctionStack_.push_back(rval);
   } else {
     // Set up to start tracking the types that will make up the
     // loop involved in the cycle.
     circularPointerTypes_.insert(circ_typ);
-    circularPointerTypeMap_[placeholder] = rval;
     assert(circularPointerLoop_.size() == 0);
     circularPointerLoop_.push_back(std::make_pair(placeholder, rval));
   }
diff --git a/bridge/go-llvm-typemanager.h b/bridge/go-llvm-typemanager.h
index 460f862..b2a9660 100644
--- a/bridge/go-llvm-typemanager.h
+++ b/bridge/go-llvm-typemanager.h
@@ -351,9 +351,6 @@
   // function types created from placeholders.
   std::unordered_set<llvm::Type *> circularFunctionTypes_;
 
-  // Stack used to help with creation of circular function types.
-  std::vector<Btype *> circularFunctionStack_;
-
   // Name generation helper
   NameGen *nametags_;
 
diff --git a/unittests/BackendCore/BackendPointerExprTests.cpp b/unittests/BackendCore/BackendPointerExprTests.cpp
index a90fb49..5b0297e 100644
--- a/unittests/BackendCore/BackendPointerExprTests.cpp
+++ b/unittests/BackendCore/BackendPointerExprTests.cpp
@@ -595,7 +595,7 @@
                                     L_RES, bi64t,
                                     L_END);
   Btype *pbefty2 = be->pointer_type(befty2);
-  be->set_placeholder_pointer_type(pht2, cft2);
+  be->set_placeholder_pointer_type(pht2, pbefty2);
   BFunctionType *befty1 = mkFuncTyp(be,
                                     L_PARM, bi64t,
                                     L_PARM, bi64t,
@@ -604,14 +604,14 @@
                                     L_RES, bi64t,
                                     L_END);
   Btype *pbefty1 = be->pointer_type(befty1);
-  be->set_placeholder_pointer_type(pht1, cft1);
+  be->set_placeholder_pointer_type(pht1, pbefty1);
 
   // Local vars
   h.mkLocal("x", pbefty1);
   h.mkLocal("y", pbefty2);
 
   const char *exp = R"RAW_RESULT(
-   store i64 (i8*, i64, i64, %CFT.0*, i64 (i8*, i64, i64, %CFT.1*, %CFT.0*)*)* null, i64 (i8*, i64, i64, %CFT.0*, i64 (i8*, i64, i64, %CFT.1*, %CFT.0*)*)** %x
+   store i64 (i8*, i64, i64, %CFT.0*, %CFT.1*)* null, i64 (i8*, i64, i64, %CFT.0*, %CFT.1*)** %x
    store i64 (i8*, i64, i64, %CFT.1*, %CFT.0*)* null, i64 (i8*, i64, i64, %CFT.1*, %CFT.0*)** %y
 
     )RAW_RESULT";