Also replace boost::detail::atomic_exchange_and_add for the interpreter.

Generalize the __cxa_atexit replacement to cover more symbols.
Because the JIT cannot handle inline asm, provide compiled symbols for relevant
functions until cling has switched to MCJIT.
This fixes the use of many boost libraries (e.g. regex) in cling on x86_64/i386.
Kudos to Marco Clemencic for the idea!

lib/Interpreter/IncrementalExecutor.cpp

@@ -20,6 +20,35 @@
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/DynamicLibrary.h"
 
+namespace {
+// From boost/detail/sp_counted_base.hpp.
+// JIT cannot handle inline asm, thus compile these symbols and
+// inject them when needed.
+static int boost__detail__atomic_exchange_and_add( int * pw, int dv )
+{
+    // int r = *pw;
+    // *pw += dv;
+    // return r;
+
+    int r;
+
+#if defined( __GNUC__ ) && ( defined( __i386__ ) || defined( __x86_64__ ) )
+    __asm__ __volatile__
+    (
+        "lock\n\t"
+        "xadd %1, %0":
+        "=m"( *pw ), "=r"( r ): // outputs (%0, %1)
+        "m"( *pw ), "1"( dv ): // inputs (%2, %3 == %1)
+        "memory", "cc" // clobbers
+    );
+#else
+    int r = *pw;
+    *pw += dv;
+#endif
+    return r;
+}
+}
+
 using namespace llvm;
 namespace cling {
 
@@ -29,7 +58,12 @@ std::vector<IncrementalExecutor::LazyFunctionCreatorFunc_t>
 
 // Keep in source: OwningPtr<ExecutionEngine> needs #include ExecutionEngine
 IncrementalExecutor::IncrementalExecutor(llvm::Module* m)
-  : m_CxaAtExitRemapped(false) {
+  : m_SymbolsToRemap{
+  {"__cxa_atexit", {0, "cling_cxa_atexit"}},
+  {"_ZN5boost6detail23atomic_exchange_and_addEPii",
+      {(void*)&boost__detail__atomic_exchange_and_add, ""}}
+  }
+{
   assert(m && "llvm::Module must not be null!");
   m_AtExitFuncs.reserve(256);
 
@@ -74,22 +108,43 @@ void IncrementalExecutor::shuttingDown() {
   }
 }
 
-void IncrementalExecutor::remapCXAAtExit() {
-  if (m_CxaAtExitRemapped)
-    return;
+void IncrementalExecutor::remapSymbols() {
+  // Note: iteration of ++remapI happens in the body due to invalidation
+  // of the erased iterator!
+  for (auto remapI = std::begin(m_SymbolsToRemap),
+         remapE = std::end(m_SymbolsToRemap);
+       remapI != remapE;) {
+    // The function for which the symbol address will be replaced
+    llvm::Function* origFunc
+      = m_engine->FindFunctionNamed(remapI->first.c_str());
+    if (!origFunc) {
+      // Go to next element.
+      ++remapI;
+      continue;
+    }
 
-  llvm::Function* atExit = m_engine->FindFunctionNamed("__cxa_atexit");
-  if (!atExit)
-    return;
+    // The new symbol address, which might be NULL to signal a symbol
+    // lookup is required
+    void* replaceAddr = remapI->second.first;
+    if (!replaceAddr) {
+      // A symbol lookup is required to find the replacement address.
+      llvm::Function* interpFunc
+        = m_engine->FindFunctionNamed(remapI->second.second.c_str());
+      assert(interpFunc && "replacement function must exist.");
+      // Generate the symbol and get its address
+      replaceAddr = m_engine->getPointerToFunction(interpFunc);
+    }
+    assert(replaceAddr && "cannot find replacement symbol");
+    // Replace the mapping of function symbol to new address
+    m_engine->updateGlobalMapping(origFunc, replaceAddr);
 
-  llvm::Function* clingAtExit
-    = m_engine->FindFunctionNamed("cling_cxa_atexit");
-  assert(clingAtExit && "cling_cxa_atexit must exist.");
-
-  void* clingAtExitAddr = m_engine->getPointerToFunction(clingAtExit);
-  assert(clingAtExitAddr && "cannot find cling_cxa_atexit");
-  m_engine->updateGlobalMapping(atExit, clingAtExitAddr);
-  m_CxaAtExitRemapped = true;
+    // Note that the current entry was successfully remapped.
+    // Save the current so we can erase it *after* the iterator increment
+    // or we would increment an invalid iterator.
+    auto remapErase = remapI;
+    ++remapI;
+    m_SymbolsToRemap.erase(remapErase);
+  }
 }
 
 void IncrementalExecutor::AddAtExitFunc(void (*func) (void*), void* arg,
@@ -168,7 +223,7 @@ IncrementalExecutor::executeFunction(llvm::StringRef funcname,
   // We don't care whether something was unresolved before.
   m_unresolvedSymbols.clear();
 
-  remapCXAAtExit();
+  remapSymbols();
 
   llvm::Function* f = m_engine->FindFunctionNamed(funcname.str().c_str());
   if (!f) {
@@ -258,7 +313,7 @@ IncrementalExecutor::runStaticInitializersOnce(llvm::Module* m) {
 
     // Execute the ctor/dtor function!
     if (llvm::Function *F = llvm::dyn_cast<llvm::Function>(FP)) {
-      remapCXAAtExit();
+      remapSymbols();
       m_engine->getPointerToFunction(F);
       // check if there is any unresolved symbol in the list
       if (!m_unresolvedSymbols.empty()) {
@@ -364,7 +419,7 @@ void* IncrementalExecutor::getAddressOfGlobal(llvm::Module* m,
     if (!gvar)
       return 0;
 
-    remapCXAAtExit();
+    remapSymbols();
     address = m_engine->getPointerToGlobal(gvar);
   }
   return address;
@@ -372,7 +427,7 @@ void* IncrementalExecutor::getAddressOfGlobal(llvm::Module* m,
 
 void*
 IncrementalExecutor::getPointerToGlobalFromJIT(const llvm::GlobalValue& GV) {
-  remapCXAAtExit();
+  remapSymbols();
   if (void* addr = m_engine->getPointerToGlobalIfAvailable(&GV))
     return addr;
 

lib/Interpreter/IncrementalExecutor.h

@@ -16,6 +16,7 @@
 
 #include <vector>
 #include <set>
+#include <map>
 
 namespace llvm {
   class ExecutionEngine;
@@ -45,9 +46,13 @@ namespace cling {
     ///
     llvm::OwningPtr<llvm::ExecutionEngine> m_engine;
 
-    ///\brief Whether cxa_at_exit has been rewired to the Interpreter's version
+    ///\brief Symbols to be replaced by special Interpreter implementations.
     ///
-    bool m_CxaAtExitRemapped;
+    /// Replaces the exectution engine's symbol "first" by second.first, or
+    /// if it is NULL, but the symbol second.second which must exist at the time
+    /// the symbol is replaced. The replacement is tried again until first us
+    /// found.
+    std::map<std::string,std::pair<void*,std::string>> m_SymbolsToRemap;
 
     ///\breif Helper that manages when the destructor of an object to be called.
     ///
@@ -175,7 +180,7 @@ namespace cling {
     ///\brief Remaps the __cxa_at_exit with a interpreter-controlled one, such
     /// that the interpreter can call the object destructors at the right time.
     ///
-    void remapCXAAtExit();
+    void remapSymbols();
 
     static void* HandleMissingFunction(const std::string&);
     static void* NotifyLazyFunctionCreators(const std::string&);