Provide __cxa_atexit replacement through custom MemoryManager.

On some platforms, global destructors are registered through a call to
__cxa_atexit(dtor, 0/*args*/, __dso_handle). While __cxa_atexit can be resolved
by the regular MemoryManager, __dso_handle (representing the "current shared
library" such that the corresponding atexit function can be called on its
dlclose) can not be resolved by MCJIT. Instead, we provide our own, poining to
the ExecutionEngine, which in turn holds a "current module" that corresponds in
spirit to the shared library handle.

__cxa_atexit, on the other hand, needs to be re-wired: the interpreter needs to
destruct globals upon its destruction, and those globals from a certain module
when that module is unloaded.

Both is done through a custom MemoryManager, significantly reducing the
complexity of the previous ("JIT without MC") implementation.

The custom MemoryManager also forwards in case of a unknown symbols to the LazyFunctionCreators instead of using the generic
ExecutionEngine::InstallLazyFunctionCreator() which has no effect with MCJIT.

include/cling/Interpreter/RuntimeUniverse.h

@@ -41,13 +41,6 @@ namespace cling {
     extern Interpreter* gCling;
 
     namespace internal {
-      ///\brief Manually provided by cling missing function resolution using
-      /// addSymbol()
-      ///
-      /// Implemented in Interpreter.cpp
-      ///
-      int local_cxa_atexit(void (*func) (void*), void* arg, void* interp);
-
       /// \brief Some of clang's routines rely on valid source locations and
       /// source ranges. This member can be looked up and source locations and
       /// ranges can be passed in as parameters to these routines.
@@ -179,27 +172,7 @@ namespace cling {
 
 using namespace cling::runtime;
 
-// Global d'tors only for C++:
-#if _WIN32
 extern "C" {
-
-  ///\brief Fake definition to avoid compilation missing function in windows
-  /// environment it wont ever be called
-  void __dso_handle(){}
-  //Fake definition to avoid compilation missing function in windows environment
-  //it wont ever be called
-  int __cxa_atexit(void (* /*func*/) (), void* /*arg*/, void* /*dso*/) {
-    return 0;
-  }
-}
-#endif
-
-extern "C" {
-  int cling_cxa_atexit(void (*func) (void*), void* arg, void* /*dso*/) {
-    return cling::runtime::internal::local_cxa_atexit(func, arg,
-                                                 (void*)cling::runtime::gCling);
-  }
-
   ///\brief a function that throws NullDerefException. This allows to 'hide' the
   /// definition of the exceptions from the RuntimeUniverse and allows us to
   /// run cling in -no-rtti mode.

lib/Interpreter/IncrementalExecutor.cpp

@@ -10,6 +10,7 @@
 #include "IncrementalExecutor.h"
 
 #include "cling/Interpreter/Value.h"
+#include "cling/Interpreter/Transaction.h"
 
 #include "clang/Basic/Diagnostic.h"
 
@@ -20,11 +21,56 @@
 #include "llvm/PassManager.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ExecutionEngine/GenericValue.h"
-#include "llvm/ExecutionEngine/JIT.h"
+#include "llvm/ExecutionEngine/MCJIT.h"
+#include "llvm/ExecutionEngine/SectionMemoryManager.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/DynamicLibrary.h"
 
 using namespace llvm;
+
+namespace {
+class ClingMemoryManager: public SectionMemoryManager {
+  cling::IncrementalExecutor* m_exe;
+  struct {
+    uint64_t __cxa_at_exit;
+  } m_addr {};
+
+  static void local_cxa_atexit(void (*func) (void*), void* arg, void* dso) {
+    cling::IncrementalExecutor* exe = (cling::IncrementalExecutor*)dso;
+    exe->AddAtExitFunc(func, arg);
+  }
+
+
+public:
+  ClingMemoryManager(cling::IncrementalExecutor* Exe):
+    m_exe(Exe) {}
+
+  ///\brief Return the address of a symbol, use callbacks if needed.
+  uint64_t getSymbolAddress (const std::string &Name) override;
+  ///\brief Simply wraps the base class's function setting AbortOnFailure
+  /// to false and instead using the error handling mechanism to report it.
+  void* getPointerToNamedFunction(const std::string &Name,
+                                  bool /*AbortOnFailure*/ =true) override {
+    return SectionMemoryManager::getPointerToNamedFunction(Name, false);
+  }
+};
+
+uint64_t ClingMemoryManager::getSymbolAddress(const std::string &Name) {
+  if (Name == "__cxa_atexit") {
+    // Rewrire __cxa_atexit to ~Interpreter(), thus also global destruction
+    // coming from the JIT.
+    return (uint64_t)&local_cxa_atexit;
+  } else if (Name == "__dso_handle") {
+    // Provide IncrementalExecutor as the third argument to __cxa_atexit.
+    return (uint64_t)m_exe;
+  }
+  if (uint64_t Addr = SectionMemoryManager::getSymbolAddress(Name))
+    return Addr;
+  return (uint64_t) m_exe->NotifyLazyFunctionCreators(Name);
+}
+
+}
+
 namespace cling {
 
 std::set<std::string> IncrementalExecutor::m_unresolvedSymbols;
@@ -34,7 +80,8 @@ std::vector<IncrementalExecutor::LazyFunctionCreatorFunc_t>
 
 // Keep in source: OwningPtr<ExecutionEngine> needs #include ExecutionEngine
   IncrementalExecutor::IncrementalExecutor(llvm::Module* m,
-                                           clang::DiagnosticsEngine& /*diags*/)
+                                           clang::DiagnosticsEngine& /*diags*/):
+    m_CurrentAtExitModule(0)
 #if 0
     : m_Diags(diags)
 #endif
@@ -42,11 +89,6 @@ std::vector<IncrementalExecutor::LazyFunctionCreatorFunc_t>
   assert(m && "llvm::Module must not be null!");
   m_AtExitFuncs.reserve(256);
 
-  // Rewrire __cxa_atexit to ~Interpreter(), thus also global destruction
-  // coming from the JIT.
-  m_SymbolsToRemap["__cxa_atexit"]
-    = std::make_pair((void*)0, std::string("cling_cxa_atexit"));
-
   //
   //  Create an execution engine to use.
   //
@@ -59,7 +101,8 @@ std::vector<IncrementalExecutor::LazyFunctionCreatorFunc_t>
   builder.setErrorStr(&errMsg);
   builder.setOptLevel(llvm::CodeGenOpt::Less);
   builder.setEngineKind(llvm::EngineKind::JIT);
-  builder.setAllocateGVsWithCode(false);
+  builder.setUseMCJIT(true);
+  builder.setMCJITMemoryManager(new ClingMemoryManager(this));
 
   // EngineBuilder uses default c'ted TargetOptions, too:
   llvm::TargetOptions TargetOpts;
@@ -72,7 +115,7 @@ std::vector<IncrementalExecutor::LazyFunctionCreatorFunc_t>
   assert(m_engine && "Cannot create module!");
 
   // install lazy function creators
-  m_engine->InstallLazyFunctionCreator(NotifyLazyFunctionCreators);
+  //m_engine->InstallLazyFunctionCreator(NotifyLazyFunctionCreators);
 }
 
 // Keep in source: ~OwningPtr<ExecutionEngine> needs #include ExecutionEngine
@@ -124,10 +167,9 @@ void IncrementalExecutor::remapSymbols() {
   }
 }
 
-void IncrementalExecutor::AddAtExitFunc(void (*func) (void*), void* arg,
-                                        const cling::Transaction* T) {
+void IncrementalExecutor::AddAtExitFunc(void (*func) (void*), void* arg) {
   // Register a CXAAtExit function
-  m_AtExitFuncs.push_back(CXAAtExitElement(func, arg, T));
+  m_AtExitFuncs.push_back(CXAAtExitElement(func, arg, m_CurrentAtExitModule));
 }
 
 void unresolvedSymbol()
@@ -237,6 +279,14 @@ IncrementalExecutor::runStaticInitializersOnce(llvm::Module* m) {
   assert(m && "Module must not be null");
   assert(m_engine && "Code generation did not create an engine!");
 
+  // Set m_CurrentAtExitModule to the Module, unset to 0 once done.
+  struct AtExitModuleSetterRAII {
+    llvm::Module*& m_AEM;
+    AtExitModuleSetterRAII(llvm::Module* M, llvm::Module*& AEM): m_AEM(AEM)
+    { AEM = M; }
+    ~AtExitModuleSetterRAII() { m_AEM = 0; }
+  } DSOHandleSetter(m, m_CurrentAtExitModule);
+
   m_engine->finalizeObject();
 
   llvm::GlobalVariable* GV
@@ -330,7 +380,7 @@ void IncrementalExecutor::runAndRemoveStaticDestructors(Transaction* T) {
   AtExitFunctions boundToT;
   for (AtExitFunctions::iterator I = m_AtExitFuncs.begin();
        I != m_AtExitFuncs.end();)
-    if (I->m_FromT == T) {
+    if (I->m_FromM == T->getModule()) {
       boundToT.push_back(*I);
       I = m_AtExitFuncs.erase(I);
     }

lib/Interpreter/IncrementalExecutor.h

@@ -83,8 +83,8 @@ namespace cling {
       ///                    atexit function.
       ///
       CXAAtExitElement(void (*func) (void*), void* arg,
-                       const Transaction* fromT):
-        m_Func(func), m_Arg(arg), m_FromT(fromT) {}
+                       const llvm::Module* fromM):
+        m_Func(func), m_Arg(arg), m_FromM(fromM) {}
 
       ///\brief The function to be called.
       ///
@@ -94,10 +94,10 @@ namespace cling {
       ///
       void* m_Arg;
 
-      ///\brief Clang's top level declaration, whose unloading will trigger the
-      /// call this atexit function.
+      ///\brief The module whose unloading will trigger the call to this atexit
+      /// function.
       ///
-      const Transaction* m_FromT; //FIXME: Should be bound to the llvm symbol.
+      const llvm::Module* m_FromM;
     };
 
     typedef llvm::SmallVector<CXAAtExitElement, 128> AtExitFunctions;
@@ -106,6 +106,11 @@ namespace cling {
     ///
     AtExitFunctions m_AtExitFuncs;
 
+    ///\brief Module for which registration of static destructors currently
+    /// takes place.
+    llvm::Module* m_CurrentAtExitModule;
+
+
 #if 0 // See FIXME in IncrementalExecutor.cpp
     ///\brief The diagnostics engine, printing out issues coming from the
     /// incremental executor.
@@ -189,8 +194,11 @@ namespace cling {
 
     ///\brief Keep track of the entities whose dtor we need to call.
     ///
-     void AddAtExitFunc(void (*func) (void*), void* arg,
-                        const cling::Transaction* clingT);
+    void AddAtExitFunc(void (*func) (void*), void* arg);
+
+    ///\brief Try to resolve a symbol through our LazyFunctionCreators;
+    /// print an error message if that fails.
+    void* NotifyLazyFunctionCreators(const std::string&);
 
   private:
     ///\brief Remaps the __cxa_at_exit with a interpreter-controlled one, such
@@ -204,7 +212,6 @@ namespace cling {
                                    llvm::StringRef title = llvm::StringRef());
 
     static void* HandleMissingFunction(const std::string&);
-    static void* NotifyLazyFunctionCreators(const std::string&);
 
   };
 } // end cling

lib/Interpreter/Interpreter.cpp

@@ -67,16 +67,6 @@ namespace {
 } // unnamed namespace
 
 namespace cling {
-  namespace runtime {
-    namespace internal {
-      // "Declared" to the JIT in RuntimeUniverse.h
-      void local_cxa_atexit(void (*func) (void*), void* arg, void* interp) {
-        Interpreter* cling = (cling::Interpreter*)interp;
-        cling->AddAtExitFunc(func, arg);
-      }
-    } // end namespace internal
-  } // end namespace runtime
-
   // FIXME: workaround until JIT supports exceptions
   jmp_buf* Interpreter::m_JumpBuf;
 
@@ -1184,7 +1174,7 @@ namespace cling {
   }
 
   void Interpreter::AddAtExitFunc(void (*Func) (void*), void* Arg) {
-    m_Executor->AddAtExitFunc(Func, Arg, getLastTransaction());
+    m_Executor->AddAtExitFunc(Func, Arg);
   }
 
   void Interpreter::GenerateAutoloadingMap(llvm::StringRef inFile,