Re-enable support for profiling/debugging interpreted/JITted code

This feature, originally added in commit 22b1606f, was reverted to
make the LLVM upgrade to version 13 easier. This commit adds back
all functionality as it was just before the LLVM upgrade.

include/cling/Utils/Utils.h

@@ -0,0 +1,38 @@
+//--------------------------------------------------------------------*- C++ -*-
+// CLING - the C++ LLVM-based InterpreterG :)
+// author: Guilherme Amadio <amadio@cern.ch>
+//
+// This file is dual-licensed: you can choose to license it under the University
+// of Illinois Open Source License or the GNU Lesser General Public License. See
+// LICENSE.TXT for details.
+//------------------------------------------------------------------------------
+
+#include <cstdlib>
+#include <cstring>
+
+#ifndef CLING_UTILS_UTILS_H
+#define CLING_UTILS_UTILS_H
+
+#if defined(_MSC_VER) && !defined(strcasecmp)
+#define strcasecmp _stricmp
+#endif
+
+namespace cling {
+  namespace utils {
+    /** Convert @p value to boolean */
+    static inline bool ConvertEnvValueToBool(const char* value) {
+      const char* true_strs[] = {"1", "true", "on", "yes"};
+
+      if (!value)
+        return false;
+
+      for (auto str : true_strs)
+        if (strcasecmp(value, str) == 0)
+          return true;
+
+      return false;
+    }
+  } // namespace utils
+} // namespace cling
+
+#endif // CLING_UTILS_UTILS_H

lib/Interpreter/CIFactory.cpp

@@ -15,6 +15,7 @@
 #include "cling/Utils/Output.h"
 #include "cling/Utils/Paths.h"
 #include "cling/Utils/Platform.h"
+#include "cling/Utils/Utils.h"
 
 #include "clang/AST/ASTContext.h"
 #include "clang/Basic/Builtins.h"
@@ -50,6 +51,7 @@
 #include "llvm/Target/TargetOptions.h"
 
 #include <cstdio>
+#include <cstdlib>
 #include <ctime>
 #include <limits>
 #include <memory>
@@ -1243,6 +1245,12 @@ namespace {
     std::vector<const char*> argvCompile(argv, argv+1);
     argvCompile.reserve(argc+32);
 
+    bool debuggingEnabled =
+        cling::utils::ConvertEnvValueToBool(std::getenv("CLING_DEBUG"));
+
+    bool profilingEnabled =
+        cling::utils::ConvertEnvValueToBool(std::getenv("CLING_PROFILE"));
+
 #if __APPLE__ && __arm64__
     argvCompile.push_back("--target=arm64-apple-darwin20.3.0");
 #endif
@@ -1323,6 +1331,16 @@ namespace {
     if(COpts.CUDAHost)
       argvCompile.push_back("--cuda-host-only");
 
+#ifdef __linux__
+    // Keep frame pointer to make JIT stack unwinding reliable for profiling
+    if (profilingEnabled)
+      argvCompile.push_back("-fno-omit-frame-pointer");
+#endif
+
+    // Disable optimizations and keep frame pointer when debugging
+    if (debuggingEnabled)
+      argvCompile.push_back("-O0 -fno-omit-frame-pointer");
+
     // argv[0] already inserted, get the rest
     argvCompile.insert(argvCompile.end(), argv+1, argv + argc);
 
@@ -1662,7 +1680,10 @@ namespace {
     // adjusted per transaction in IncrementalParser::codeGenTransaction().
     CGOpts.setInlining(CodeGenOptions::NormalInlining);
 
-    // CGOpts.setDebugInfo(clang::CodeGenOptions::FullDebugInfo);
+    // Add debugging info when debugging or profiling
+    if (debuggingEnabled || profilingEnabled)
+      CGOpts.setDebugInfo(codegenoptions::FullDebugInfo);
+
     // CGOpts.EmitDeclMetadata = 1; // For unloading, for later
     // aliasing the complete ctor to the base ctor causes the JIT to crash
     CGOpts.CXXCtorDtorAliases = 0;

lib/Interpreter/CMakeLists.txt

@@ -87,6 +87,7 @@ add_cling_library(clingInterpreter OBJECT
   InvocationOptions.cpp
   LookupHelper.cpp
   NullDerefProtectionTransformer.cpp
+  PerfJITEventListener.cpp
   RequiredSymbols.cpp
   Transaction.cpp
   TransactionUnloader.cpp

lib/Interpreter/IncrementalJIT.cpp

@@ -13,6 +13,7 @@
 #include "IncrementalExecutor.h"
 
 #include "cling/Utils/Output.h"
+#include "cling/Utils/Utils.h"
 
 #include <llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h>
 #include <llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h>
@@ -286,6 +287,9 @@ Error RTDynamicLibrarySearchGenerator::tryToGenerate(
 
 namespace cling {
 
+///\brief Creates JIT event listener to allow profiling of JITted code with perf
+llvm::JITEventListener* createPerfJITEventListener();
+
 IncrementalJIT::IncrementalJIT(
     IncrementalExecutor& Executor, std::unique_ptr<TargetMachine> TM,
     std::unique_ptr<llvm::orc::ExecutorProcessControl> EPC, Error& Err,
@@ -314,6 +318,16 @@ IncrementalJIT::IncrementalJIT(
     auto Layer =
         std::make_unique<RTDyldObjectLinkingLayer>(ES, std::move(GetMemMgr));
 
+    // Register JIT event listeners if enabled
+    if (cling::utils::ConvertEnvValueToBool(std::getenv("CLING_DEBUG")))
+      Layer->registerJITEventListener(
+          *JITEventListener::createGDBRegistrationListener());
+
+#ifdef __linux__
+    if (cling::utils::ConvertEnvValueToBool(std::getenv("CLING_PROFILE")))
+      Layer->registerJITEventListener(*cling::createPerfJITEventListener());
+#endif
+
     // The following is based on LLJIT::createObjectLinkingLayer.
     if (TT.isOSBinFormatCOFF()) {
       Layer->setOverrideObjectFlagsWithResponsibilityFlags(true);

lib/Interpreter/PerfJITEventListener.cpp

@@ -0,0 +1,129 @@
+//--------------------------------------------------------------------*- C++ -*-
+// CLING - the C++ LLVM-based InterpreterG :)
+// author: Guilherme Amadio <amadio@cern.ch>
+//
+// This file is dual-licensed: you can choose to license it under the University
+// of Illinois Open Source License or the GNU Lesser General Public License. See
+// LICENSE.TXT for details.
+//------------------------------------------------------------------------------
+//
+// This file implements a JITEventListener object that tells perf about JITted
+// symbols using perf map files (/tmp/perf-%d.map, where %d = pid of process).
+//
+// Documentation for this perf jit interface is available at:
+// https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/tree/tools/perf/Documentation/jit-interface.txt
+//
+//------------------------------------------------------------------------------
+
+#ifdef __linux__
+
+#include "llvm/ExecutionEngine/JITEventListener.h"
+#include "llvm/Object/ObjectFile.h"
+#include "llvm/Object/SymbolSize.h"
+#include "llvm/Support/ManagedStatic.h"
+
+#include <cstdint>
+#include <cstdio>
+#include <mutex>
+
+#include <unistd.h>
+
+using namespace llvm;
+using namespace llvm::object;
+
+namespace {
+
+  class PerfJITEventListener : public JITEventListener {
+  public:
+    PerfJITEventListener();
+    ~PerfJITEventListener() {
+      if (m_Perfmap)
+        fclose(m_Perfmap);
+    }
+
+    void notifyObjectLoaded(ObjectKey K, const ObjectFile& Obj,
+                            const RuntimeDyld::LoadedObjectInfo& L) override;
+    void notifyFreeingObject(ObjectKey K) override;
+
+  private:
+    std::mutex m_Mutex;
+    FILE* m_Perfmap;
+  };
+
+  PerfJITEventListener::PerfJITEventListener() {
+    char filename[64];
+    snprintf(filename, 64, "/tmp/perf-%d.map", getpid());
+    m_Perfmap = fopen(filename, "a");
+  }
+
+  void PerfJITEventListener::notifyObjectLoaded(
+      ObjectKey K, const ObjectFile& Obj,
+      const RuntimeDyld::LoadedObjectInfo& L) {
+
+    if (!m_Perfmap)
+      return;
+
+    OwningBinary<ObjectFile> DebugObjOwner = L.getObjectForDebug(Obj);
+    const ObjectFile& DebugObj = *DebugObjOwner.getBinary();
+
+    // For each symbol, we want to check its address and size
+    // if it's a function and write the information to the perf
+    // map file, otherwise we just ignore the symbol and any
+    // related errors. This implementation is adapted from LLVM:
+    // llvm/src/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp
+
+    for (const std::pair<SymbolRef, uint64_t>& P :
+         computeSymbolSizes(DebugObj)) {
+      SymbolRef Sym = P.first;
+
+      Expected<SymbolRef::Type> SymTypeOrErr = Sym.getType();
+      if (!SymTypeOrErr) {
+        consumeError(SymTypeOrErr.takeError());
+        continue;
+      }
+
+      SymbolRef::Type SymType = *SymTypeOrErr;
+      if (SymType != SymbolRef::ST_Function)
+        continue;
+
+      Expected<StringRef> Name = Sym.getName();
+      if (!Name) {
+        consumeError(Name.takeError());
+        continue;
+      }
+
+      Expected<uint64_t> AddrOrErr = Sym.getAddress();
+      if (!AddrOrErr) {
+        consumeError(AddrOrErr.takeError());
+        continue;
+      }
+
+      uint64_t address = *AddrOrErr;
+      uint64_t size = P.second;
+
+      if (size == 0)
+        continue;
+
+      std::lock_guard<std::mutex> lock(m_Mutex);
+      fprintf(m_Perfmap, "%" PRIx64 " %" PRIx64 " %s\n", address, size,
+              Name->data());
+    }
+
+    fflush(m_Perfmap);
+  }
+
+  void PerfJITEventListener::notifyFreeingObject(ObjectKey K) {
+    // nothing to be done
+  }
+
+  llvm::ManagedStatic<PerfJITEventListener> PerfListener;
+
+} // end anonymous namespace
+
+namespace cling {
+
+  JITEventListener* createPerfJITEventListener() { return &*PerfListener; }
+
+} // namespace cling
+
+#endif