From d4e822e8d9dac88c4a2d04bac187904b7e9500fc Mon Sep 17 00:00:00 2001 From: Vassil Vassilev Date: Mon, 21 Oct 2019 15:41:12 +0200 Subject: [PATCH] Sink {libc,std}.modulemap in cling. This patch teaches cling to detect if the essential libraries have modulemaps and if necessary it adds an overlay around libc and std. This tightens the implementation and makes cling standalone easier to run in -fmodules mode. --- include/cling/Interpreter/ClingOptions.inc | 2 - include/cling/Interpreter/InvocationOptions.h | 3 - include/cling/libc.modulemap | 68 +++ include/cling/std.modulemap | 430 ++++++++++++++++++ lib/Interpreter/CIFactory.cpp | 179 ++++---- lib/Interpreter/InvocationOptions.cpp | 3 - 6 files changed, 585 insertions(+), 100 deletions(-) create mode 100644 include/cling/libc.modulemap create mode 100644 include/cling/std.modulemap diff --git a/include/cling/Interpreter/ClingOptions.inc b/include/cling/Interpreter/ClingOptions.inc index ec57d9dd..55e507fe 100644 --- a/include/cling/Interpreter/ClingOptions.inc +++ b/include/cling/Interpreter/ClingOptions.inc @@ -23,8 +23,6 @@ OPTION(prefix_2, "errorout", _errorout, Flag, INVALID, INVALID, 0, 0, 0, // Re-implement to forward to our help OPTION(prefix_3, "help", help, Flag, INVALID, INVALID, 0, 0, 0, "Print this help text", 0, 0) -OPTION(prefix_1, "includedir_loc=", overlay_EQ, Joined, INVALID, INVALID, 0, 0, 0, - "Set the Modules overlay file", 0, 0) OPTION(prefix_1, "L", L, JoinedOrSeparate, INVALID, INVALID, 0, 0, 0, "Add directory to library search path", "", 0) OPTION(prefix_1, "l", l, JoinedOrSeparate, INVALID, INVALID, 0, 0, 0, diff --git a/include/cling/Interpreter/InvocationOptions.h b/include/cling/Interpreter/InvocationOptions.h index 512ba6f8..55827bf1 100644 --- a/include/cling/Interpreter/InvocationOptions.h +++ b/include/cling/Interpreter/InvocationOptions.h @@ -86,9 +86,6 @@ namespace cling { /// directive for the MetaProcessor. Defaults to "." std::string MetaString; - /// \brief A path of modulemaps to be overlayed. - std::string OverlayFile; - std::vector LibsToLoad; std::vector LibSearchPath; std::vector Inputs; diff --git a/include/cling/libc.modulemap b/include/cling/libc.modulemap new file mode 100644 index 00000000..429e9e8b --- /dev/null +++ b/include/cling/libc.modulemap @@ -0,0 +1,68 @@ +module "libc" [system] [extern_c] [no_undeclared_includes] { + export * + module "assert.h" { + export * + textual header "assert.h" + } + module "string.h" { + export * + header "string.h" + } + module "ctype.h" { + export * + header "ctype.h" + } + module "errno.h" { + export * + header "errno.h" + } + module "fenv.h" { + export * + header "fenv.h" + } + module "inttypes.h" { + export * + header "inttypes.h" + } + module "math.h" { + export * + header "math.h" + } + module "pthread.h" { + export * + header "pthread.h" + } + module "setjmp.h" { + export * + header "setjmp.h" + } + module "signal.h" { + export * + header "signal.h" + } + module "stdio.h" { + export * + header "stdio.h" + } + module "stdlib.h" { + export * + header "stdlib.h" + } + module "time.h" { + export * + header "time.h" + } + module "wchar.h" { + export * + header "wchar.h" + } + + use "xlocale.h" +} + +// glib specific header. In it's own module because it +// doesn't exist on some systems with unpatched glib 2.26+ +module "xlocale.h" [system] [extern_c] { + export * + header "xlocale.h" +} diff --git a/include/cling/std.modulemap b/include/cling/std.modulemap new file mode 100644 index 00000000..febea6a7 --- /dev/null +++ b/include/cling/std.modulemap @@ -0,0 +1,430 @@ +module "std" [system] { + export * + module "algorithm" { + export * + header "algorithm" + } + module "array" { + export * + header "array" + } + module "atomic" { + export * + header "atomic" + } + module "bitset" { + export * + header "bitset" + } + // no module for cassert to stay consistent with the OS X modulemap + module "ccomplex" { + export * + header "ccomplex" + } + module "cctype" { + export * + header "cctype" + } + module "cerrno" { + export * + header "cerrno" + } + module "cfenv" { + export * + header "cfenv" + } + module "cfloat" { + export * + header "cfloat" + } + module "chrono" { + export * + header "chrono" + } + module "cinttypes" { + export * + header "cinttypes" + } + module "ciso646" { + export * + header "ciso646" + } + module "climits" { + export * + header "climits" + } + module "clocale" { + export * + header "clocale" + } + module "cmath" { + export * + header "cmath" + } + module "complex" { + export * + header "complex" + } + module "complex.h" { + export * + header "complex.h" + } + module "condition_variable" { + export * + header "condition_variable" + } + module "csetjmp" { + export * + header "csetjmp" + } + module "csignal" { + export * + header "csignal" + } + module "cstdalign" { + requires !cplusplus17, !header_existence + export * + header "cstdalign" + } + module "cstdarg" { + export * + header "cstdarg" + } + module "cstdbool" { + export * + header "cstdbool" + } + module "cstddef" { + export * + header "cstddef" + } + module "cstdint" { + export * + header "cstdint" + } + module "cstdio" { + export * + header "cstdio" + } + // Causes a cycle between clang's builtin modules + // and the STL as clang's builtin modules include + // this header (and in turn, the STL also includes + // clang's builtin headers). + // See the include for stdlib.h (which is forwarded + // to this C++ header) in clang's mm_malloc.h for the + // problematic code which we can't fix. + module "cstdlib" { + export * + textual header "cstdlib" + } + module "cstring" { + export * + header "cstring" + } + module "ctgmath" { + export * + header "ctgmath" + } + module "ctime" { + export * + header "ctime" + } +// module "ctype.h" { +// export * +// header "ctype.h" +// } + module "cwchar" { + export * + header "cwchar" + } + module "cwctype" { + export * + header "cwctype" + } + module "cxxabi.h" { + export * + header "cxxabi.h" + } + module "deque" { + export * + header "deque" + } + module "exception" { + export * + header "exception" + } + module "fenv.h" { + export * + header "fenv.h" + } + module "forward_list" { + export * + header "forward_list" + } + module "fstream" { + export * + header "fstream" + } + module "functional" { + export * + header "functional" + } + module "future" { + export * + header "future" + } +/* module "hash_map" { + export * + header "hash_map" + } + module "hash_set" { + export * + header "hash_set" + } + */ + module "initializer_list" { + export * + header "initializer_list" + } + module "iomanip" { + export * + header "iomanip" + } + module "ios" { + export * + header "ios" + } + module "iosfwd" { + export * + header "iosfwd" + } + module "iostream" { + export * + header "iostream" + } +// module "iostream.h" { +// export * +// header "iostream.h" +// } + module "istream" { + export * + header "istream" + } + module "iterator" { + export * + header "iterator" + } + module "limits" { + export * + header "limits" + } + module "list" { + export * + header "list" + } + module "locale" { + export * + header "locale" + } +// module "locale.h" { +// export * +// header "locale.h" +// } + module "map" { + export * + header "map" + } +// module "math.h" { +// export * +// header "math.h" +// } + module "memory" { + export * + header "memory" + } + module "mutex" { + export * + header "mutex" + } + module "new" { + export * + header "new" + } + module "numeric" { + export * + header "numeric" + } + module "ostream" { + export * + header "ostream" + } + module "queue" { + export * + header "queue" + } + module "random" { + export * + header "random" + } + module "ratio" { + export * + header "ratio" + } + module "regex" { + export * + header "regex" + } + module "scoped_allocator" { + export * + header "scoped_allocator" + } + module "set" { + export * + header "set" + } +// module "setjmp.h" { +// export * +// header "setjmp.h" +// } + module "sstream" { + export * + header "sstream" + } + module "stack" { + export * + header "stack" + } + module "stdexcept" { + export * + header "stdexcept" + } + module "streambuf" { + export * + header "streambuf" + } + module "string" { + export * + header "string" + } + module "string_view" { + requires !header_existence + export * + textual header "string_view" + } +// module "string.h" { +// export * +// header "string.h" + // } + module "system_error" { + export * + header "system_error" + } +// module "tgmath.h" { +// export * +// header "tgmath.h" +// } + module "thread" { + export * + header "thread" + } + module "tuple" { + export * + header "tuple" + } + module "type_traits" { + export * + header "type_traits" + } + module "typeindex" { + export * + header "typeindex" + } + module "typeinfo" { + export * + header "typeinfo" + } + module "unordered_map" { + export * + header "unordered_map" + } + module "unordered_set" { + export * + header "unordered_set" + } + module "utility" { + export * + header "utility" + } + module "valarray" { + export * + header "valarray" + } + module "vector" { + export * + header "vector" + } + module "codecvt" { + requires !header_existence + export * + header "codecvt" + } + module "cuchar" { + requires !header_existence + export * + header "cuchar" + } + //module "experimental/algorithm" { + // export * + // header "experimental/algorithm" + //} + module "ext/functional" { + export * + header "ext/functional" + } +/* module "ext/hash_map" { + export * + header "ext/hash_map" + } + module "ext/hash_set" { + export * + header "ext/hash_set" + } +*/ + module "ext/numeric" { + export * + header "ext/numeric" + } + module "bits/allocator.h" { + export * + header "bits/allocator.h" + } + module "bits/basic_ios.h" { + export * + header "bits/basic_ios.h" + } + module "bits/cpp_type_traits.h" { + export * + header "bits/cpp_type_traits.h" + } + module "bits/exception_defines.h" { + export * + header "bits/exception_defines.h" + } + module "bits/ios_base.h" { + export * + header "bits/ios_base.h" + } + module "bits/locale_facets.h" { + export * + header "bits/locale_facets.h" + } + module "bits/stl_map.h" { + export * + export bits_stl_tree_h + header "bits/stl_map.h" + } + explicit module "bits_stl_tree_h" { + export * + header "bits/stl_tree.h" + } +} diff --git a/lib/Interpreter/CIFactory.cpp b/lib/Interpreter/CIFactory.cpp index b36a874d..a9d6287a 100644 --- a/lib/Interpreter/CIFactory.cpp +++ b/lib/Interpreter/CIFactory.cpp @@ -557,88 +557,87 @@ namespace { } } - static void locateLibC(llvm::SmallVectorImpl& loc) { - // FIXME: Support system which doesn't have /usr/include as libc path. - // We need to find out how to identify the correct libc path on such system. - llvm::sys::path::append(loc, llvm::sys::path::get_separator(), - "usr", "include"); - } + static std::string getIncludePathForHeader(clang::HeaderSearchOptions& Opts, + llvm::StringRef header) { + for (unsigned i = 0, e = Opts.UserEntries.size(); i != e; ++i) { + // FIXME: We should compare against the -cxx-isystem, however, that can + // only happen if we go through the clang toolchain infrastructure. + // Currently we ask the system compiler about the include paths and we + // do not have information what is -cxx-isystem and -c-isystem... + // if (Opts.UserEntries[i].Group != Group) + // continue; - static void locateLibStd(llvm::SmallString<256>& loc, - const HeaderSearch& HS) { - // Check if the system path exists. If it does and it contains - // "/include/c++/" (as stl path is always inferred from gcc path). - // FIXME: Implement a more sophisticated way to detect stl paths - for (auto I = HS.system_dir_begin(), E = HS.system_dir_end(); I != E; ++I) { - if (!I->getDir()) - continue; - if (llvm::sys::path::filename(I->getName()) == "backward") - continue; - if (!I->getName().contains("/include/c++/")) - continue; - - loc = I->getName(); - break; + llvm::SmallString<512> headerPath(Opts.UserEntries[i].Path); + llvm::sys::path::append(headerPath, header); + if (llvm::sys::fs::exists(headerPath.str())) + return Opts.UserEntries[i].Path; } - assert(loc.size() && "Failed to locate std."); + return {}; } - static void collectModuleMaps(clang::CompilerInvocation& Invocation, - const HeaderSearch& HS, - llvm::StringRef OverlayFileLoc, + static void collectModuleMaps(clang::CompilerInstance& CI, llvm::SmallVectorImpl &ModuleMapFiles) { - assert(Invocation.getLangOpts()->Modules && + assert(CI.getLangOpts().Modules && "Using overlay without -fmodules"); - llvm::SmallString<128> libCLoc; - locateLibC(libCLoc); + clang::HeaderSearchOptions& HSOpts = CI.getHeaderSearchOpts(); - llvm::SmallString<256> libStdLoc; - locateLibStd(libStdLoc, HS); + // FIXME: Implement CLING_C_INCL similar to CLING_CXX_INCL and then we can + // be more independent on the location of libc. + //llvm::SmallString<128> libCLoc(getIncludePathForHeader(HSOpts, "assert.h")); - if (!OverlayFileLoc.empty()) { + llvm::SmallString<128> cIncLoc("/usr/include"); - // Construct a column of modulemap overlay file, given System filename, - // Location + Filename (modulemap to be overlayed). If NotLast is true, - // append ",". - auto buildModuleMapOverlayEntry = [](llvm::StringRef SystemDir, - const std::string& Filename, - const std::string& Location, - bool Last = false) { - llvm::SmallString<512> originalLoc(Location); - llvm::sys::path::append(originalLoc, Filename); + llvm::SmallString<256> stdIncLoc(getIncludePathForHeader(HSOpts, "cassert")); - llvm::SmallString<512> systemLoc(SystemDir); - llvm::sys::path::append(systemLoc, "module.modulemap"); - // Check if we have already a modulemap at the location where we will - // create a virtual modulemap file. This can happen when we are on - // linux but using libc++. - if (llvm::sys::fs::exists(systemLoc.str())) { - llvm::errs() << "Modulemap " << systemLoc.str() - << " already exists! Replacing it with " + llvm::SmallString<256> clingIncLoc(getIncludePathForHeader(HSOpts, + "cling/Interpreter/RuntimeUniverse.h")); + + // Re-add cling as the modulemap are in cling/*modulemap + llvm::sys::path::append(clingIncLoc, "cling"); + + // Construct a column of modulemap overlay file if needed. + auto maybeAppendOverlayEntry = [&HSOpts](llvm::StringRef SystemDir, + const std::string& Filename, + const std::string& Location, + std::string& overlay) { + llvm::SmallString<512> originalLoc(Location); + llvm::sys::path::append(originalLoc, Filename); + + assert(llvm::sys::fs::exists(originalLoc.str()) && "Must exist!"); + + llvm::SmallString<512> systemLoc(SystemDir); + llvm::sys::path::append(systemLoc, "module.modulemap"); + // Check if we need to mount a custom modulemap. We may have it, for + // instance when we are on osx or using libc++. + if (llvm::sys::fs::exists(systemLoc.str())) { + if (HSOpts.Verbose) + cling::log() << systemLoc.str() + << " already exists! Skip replacing it with " << originalLoc.str(); - } + return; + } - std::string overlay; - overlay += "{ 'name': '" + SystemDir.str() + "', 'type': 'directory',\n"; - overlay += "'contents': [\n { 'name': 'module.modulemap', "; - overlay += "'type': 'file',\n 'external-contents': '"; - overlay += originalLoc.str().str() + "'\n"; - overlay += "}\n ]\n }"; - if (!Last) - overlay += ",\n"; + if (!overlay.empty()) + overlay += ",\n"; - return overlay; - }; + overlay += "{ 'name': '" + SystemDir.str() + "', 'type': 'directory',\n"; + overlay += "'contents': [\n { 'name': 'module.modulemap', "; + overlay += "'type': 'file',\n 'external-contents': '"; + overlay += originalLoc.str().str() + "'\n"; + overlay += "}\n ]\n }"; + }; + std::string MOverlay; + maybeAppendOverlayEntry(cIncLoc.str(), "libc.modulemap", clingIncLoc.str(), + MOverlay); + maybeAppendOverlayEntry(stdIncLoc.str(), "std.modulemap", clingIncLoc.str(), + MOverlay); + + if (/*needsOverlay*/!MOverlay.empty()) { // Virtual modulemap overlay file - std::string MOverlay = "{\n 'version': 0,\n 'roots': [\n"; + MOverlay.insert(0, "{\n 'version': 0,\n 'roots': [\n"); - MOverlay += buildModuleMapOverlayEntry(libCLoc, "libc.modulemap", - OverlayFileLoc); - MOverlay += buildModuleMapOverlayEntry(libStdLoc, "std.modulemap", - OverlayFileLoc, - /*Last*/ true); MOverlay += "]\n }\n ]\n }\n"; // Set up the virtual modulemap overlay file @@ -651,10 +650,9 @@ namespace { llvm::errs() << "Error in modulemap.overlay!\n"; // Load virtual modulemap overlay file - Invocation.addOverlay(FS); + CI.getInvocation().addOverlay(FS); } - clang::HeaderSearchOptions& HSOpts = HS.getHeaderSearchOpts(); if (HSOpts.ImplicitModuleMaps) return; @@ -662,17 +660,17 @@ namespace { llvm::SmallString<512> resourceDirLoc(HSOpts.ResourceDir); llvm::sys::path::append(resourceDirLoc, "include", "module.modulemap"); ModuleMapFiles.push_back(resourceDirLoc.str().str()); - llvm::sys::path::append(libCLoc, "module.modulemap"); - ModuleMapFiles.push_back(libCLoc.str().str()); - llvm::sys::path::append(libStdLoc, "module.modulemap"); - ModuleMapFiles.push_back(libStdLoc.str().str()); + llvm::sys::path::append(cIncLoc, "module.modulemap"); + ModuleMapFiles.push_back(cIncLoc.str().str()); + llvm::sys::path::append(stdIncLoc, "module.modulemap"); + ModuleMapFiles.push_back(stdIncLoc.str().str()); + llvm::sys::path::append(clingIncLoc, "module.modulemap"); + ModuleMapFiles.push_back(clingIncLoc.str().str()); } - static void setupCxxModules(clang::CompilerInvocation& Invocation, - const HeaderSearch &HS, - llvm::StringRef OverlayFileLoc) { - assert(Invocation.getLangOpts()->Modules); - clang::HeaderSearchOptions& HSOpts = HS.getHeaderSearchOpts(); + static void setupCxxModules(clang::CompilerInstance& CI) { + assert(CI.getLangOpts().Modules); + clang::HeaderSearchOptions& HSOpts = CI.getHeaderSearchOpts(); // Register prebuilt module paths where we will lookup module files. addPrebuiltModulePaths(HSOpts, getPathsFromEnv(getenv("CLING_PREBUILT_MODULE_PATH"))); @@ -682,13 +680,13 @@ namespace { // of modulemap files to load. llvm::SmallVector ModuleMaps; - collectModuleMaps(Invocation, HS, OverlayFileLoc, ModuleMaps); + collectModuleMaps(CI, ModuleMaps); assert(HSOpts.ImplicitModuleMaps == ModuleMaps.empty() && "We must have register the modulemaps by hand!"); // Prepend the modulemap files we attached so that they will be loaded. if (!HSOpts.ImplicitModuleMaps) { - FrontendOptions& FrontendOpts = Invocation.getFrontendOpts(); + FrontendOptions& FrontendOpts = CI.getInvocation().getFrontendOpts(); FrontendOpts.ModuleMapFiles.insert(FrontendOpts.ModuleMapFiles.begin(), ModuleMaps.begin(), ModuleMaps.end()); } @@ -915,7 +913,7 @@ static void stringifyPreprocSetting(PreprocessorOptions& PPOpts, static CompilerInstance* createCIImpl(std::unique_ptr Buffer, - const CompilerOptions& COpts, const std::string& ModuleMapLoc, + const CompilerOptions& COpts, const char* LLVMDir, std::unique_ptr customConsumer, const CIFactory::ModuleFileExtensions& moduleExtensions, @@ -1111,6 +1109,15 @@ static void stringifyPreprocSetting(PreprocessorOptions& PPOpts, return CI.release(); } + // With modules, we now start adding prebuilt module paths to the CI. + // Modules from those paths are treated like they are never out of date + // and we don't update them on demand. + // This mostly helps ROOT where we can't just recompile any out of date + // modules because we would miss the annotations that rootcling creates. + if (COpts.CxxModules) { + setupCxxModules(*CI); + } + CI->createFileManager(); clang::CompilerInvocation& Invocation = CI->getInvocation(); std::string& PCHFile = Invocation.getPreprocessorOpts().ImplicitPCHInclude; @@ -1245,16 +1252,6 @@ static void stringifyPreprocSetting(PreprocessorOptions& PPOpts, CI->createPreprocessor(TU_Complete); Preprocessor& PP = CI->getPreprocessor(); - // With modules, we now start adding prebuilt module paths to the CI. - // Modules from those paths are treated like they are never out of date - // and we don't update them on demand. - // This mostly helps ROOT where we can't just recompile any out of date - // modules because we would miss the annotations that rootcling creates. - if (COpts.CxxModules) { - setupCxxModules(Invocation, PP.getHeaderSearchInfo(), ModuleMapLoc); - } - - PP.getBuiltinInfo().initializeBuiltins(PP.getIdentifierTable(), PP.getLangOpts()); @@ -1394,7 +1391,6 @@ CIFactory::createCI(llvm::StringRef Code, const InvocationOptions& Opts, std::unique_ptr consumer, const ModuleFileExtensions& moduleExtensions) { return createCIImpl(llvm::MemoryBuffer::getMemBuffer(Code), Opts.CompilerOpts, - Opts.OverlayFile, LLVMDir, std::move(consumer), moduleExtensions, false /*OnlyLex*/, !Opts.IsInteractive()); @@ -1404,8 +1400,7 @@ CompilerInstance* CIFactory::createCI( MemBufPtr_t Buffer, int argc, const char* const* argv, const char* LLVMDir, std::unique_ptr consumer, const ModuleFileExtensions& moduleExtensions, bool OnlyLex /*false*/) { - return createCIImpl(std::move(Buffer), CompilerOptions(argc, argv), - /*OverlayLoc*/"", LLVMDir, + return createCIImpl(std::move(Buffer), CompilerOptions(argc, argv), LLVMDir, std::move(consumer), moduleExtensions, OnlyLex); } diff --git a/lib/Interpreter/InvocationOptions.cpp b/lib/Interpreter/InvocationOptions.cpp index 2e1021f7..6ee95b32 100644 --- a/lib/Interpreter/InvocationOptions.cpp +++ b/lib/Interpreter/InvocationOptions.cpp @@ -82,9 +82,6 @@ static const char kNoStdInc[] = "-nostdinc"; Opts.MetaString = "."; } } - - if (const Arg* OverlayFileArg = Args.getLastArg(OPT_overlay_EQ)) - Opts.OverlayFile = OverlayFileArg->getValue(); } static void Extend(std::vector& A, std::vector B) {