gollvm: wrap memmove, etc. in nosplit functions
When memmove (memcpy, memset) is called from a split-stack
function, the linker will change it to force a split by calling
__morestack_non_split, because the libc functions are external
and the linker has no knowledge about them. This may cause
many unnecessary stack splits.
However, the linker is happy if they are called from a nosplit
function. So we can work around it by wrapping memmove, etc. in
nosplit functions.
This CL adds nosplit wrappers functions, __go_memmove, etc.,
which (tail) calls memmove. Also add a LLVM backend pass, runs at
the end, changing calls to memmove, etc. to call the wrapper
functions.
Change-Id: Ic7e39281275ff7da14a8744ee56b4dc57c07321d
Reviewed-on: https://go-review.googlesource.com/c/gollvm/+/182817
Reviewed-by: Than McIntosh <thanm@google.com>
diff --git a/driver/CompileGo.cpp b/driver/CompileGo.cpp
index 5eba85e..dc23698 100644
--- a/driver/CompileGo.cpp
+++ b/driver/CompileGo.cpp
@@ -961,6 +961,7 @@
passConfig->setInitialized();
codeGenPasses.add(createGoNilChecksPass());
+ codeGenPasses.add(createGoWrappersPass());
if (enable_gc_)
codeGenPasses.add(createGoAnnotationPass());
diff --git a/libgo/CMakeLists.txt b/libgo/CMakeLists.txt
index bd27903..f47ede2 100644
--- a/libgo/CMakeLists.txt
+++ b/libgo/CMakeLists.txt
@@ -449,6 +449,9 @@
list(APPEND runtimecpaths "${libgo_csrcroot}/${cfile}")
endforeach()
+# go-wrapper.c is not in gofrontend/libgo
+list(APPEND runtimecpaths "${GOLLVM_SOURCE_DIR}/libgo/runtime/go-wrappers.c")
+
# Compiler flags for C files in the runtime.
set(baseopts "-g -Wno-zero-length-array ")
if(GOLLVM_USE_SPLIT_STACK)
diff --git a/libgo/runtime/go-wrappers.c b/libgo/runtime/go-wrappers.c
new file mode 100644
index 0000000..6110894
--- /dev/null
+++ b/libgo/runtime/go-wrappers.c
@@ -0,0 +1,36 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Libc function wrappers built with split-stack. So the
+// linker won't patch the caller to force a stack split.
+
+#include <stddef.h>
+#include "unwind.h"
+
+void *__go_memmove(void*, const void*, size_t) __attribute__((no_split_stack));
+void *__go_memmove(void *dst, const void *src, size_t n)
+{
+ return __builtin_memmove(dst, src, n);
+}
+
+void *__go_memcpy(void*, const void*, size_t) __attribute__((no_split_stack));
+void *__go_memcpy(void *dst, const void *src, size_t n)
+{
+ return __builtin_memcpy(dst, src, n);
+}
+
+void *__go_memset(void*, int, size_t) __attribute__((no_split_stack));
+void *__go_memset(void *dst, int c, size_t n)
+{
+ return __builtin_memset(dst, c, n);
+}
+
+// Not no_split_stack -- _Unwind_Resume does use some stack.
+// This wrapper is useful in that it forces a split in here, not in
+// the caller. So the forced split happens only when _Unwind_Resume
+// is actually called.
+void __go_Unwind_Resume (struct _Unwind_Exception *e)
+{
+ _Unwind_Resume(e);
+}
diff --git a/passes/CMakeLists.txt b/passes/CMakeLists.txt
index 468b3b6..22a0a3b 100644
--- a/passes/CMakeLists.txt
+++ b/passes/CMakeLists.txt
@@ -11,6 +11,7 @@
GoAnnotation.cpp
GoNilChecks.cpp
GoStatepoints.cpp
+ GoWrappers.cpp
RemoveAddrSpace.cpp
Util.cpp
diff --git a/passes/GoWrappers.cpp b/passes/GoWrappers.cpp
new file mode 100644
index 0000000..0972a1f
--- /dev/null
+++ b/passes/GoWrappers.cpp
@@ -0,0 +1,93 @@
+//===--- GoWrappers.cpp ---------------------------------------------------===//
+//
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+//
+//===----------------------------------------------------------------------===//
+//
+// Replace calls to some libc functions (e.g. memmove)
+// with wrappers __go_memmove, etc., for better split-stack
+// performance.
+//
+//===----------------------------------------------------------------------===//
+
+#include "GollvmPasses.h"
+
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/TargetInstrInfo.h"
+#include "llvm/PassRegistry.h"
+#include "llvm/Target/TargetMachine.h"
+
+using namespace llvm;
+
+static cl::opt<bool> Disabled("disable-go-wrappers",
+ cl::desc("Disable Go wrappers pass"),
+ cl::init(false), cl::Hidden);
+
+namespace {
+
+class GoWrappers : public MachineFunctionPass {
+ public:
+ static char ID;
+
+ GoWrappers() : MachineFunctionPass(ID) {
+ initializeGoWrappersPass(*PassRegistry::getPassRegistry());
+ }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.setPreservesAll();
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+
+ bool runOnMachineFunction(MachineFunction &MF) override;
+
+ private:
+};
+
+} // namespace
+
+char GoWrappers::ID = 0;
+INITIALIZE_PASS(GoWrappers, "go-wrappers",
+ "Replace libc calls with split-stack wrappers", false,
+ false)
+FunctionPass *llvm::createGoWrappersPass() { return new GoWrappers(); }
+
+bool
+GoWrappers::runOnMachineFunction(MachineFunction &MF) {
+ if (Disabled)
+ return false;
+
+ for (MachineBasicBlock &MBB : MF)
+ for (MachineInstr &MI : MBB)
+ if (MI.isCall()) {
+ // memmove is an external symbol, whereas _Unwind_Resume
+ // is a global address. We handle both cases. It is fine
+ // that we always replace with external symbol.
+ const char *Name;
+ if (MI.getOperand(0).isSymbol())
+ Name = MI.getOperand(0).getSymbolName();
+ else if (MI.getOperand(0).isGlobal())
+ Name = MI.getOperand(0).getGlobal()->getName().str().c_str();
+ else
+ continue;
+
+ const char *NewName;
+ if (strcmp(Name, "memmove") == 0)
+ NewName = "__go_memmove";
+ else if (strcmp(Name, "memcpy") == 0)
+ NewName = "__go_memcpy";
+ else if (strcmp(Name, "memset") == 0)
+ NewName = "__go_memset";
+ else if (strcmp(Name, "_Unwind_Resume") == 0)
+ NewName = "__go_Unwind_Resume";
+ else
+ continue;
+
+ unsigned Flags = MI.getOperand(0).getTargetFlags();
+ MI.getOperand(0).ChangeToES(NewName, Flags);
+ }
+
+ return true;
+}
diff --git a/passes/GollvmPasses.h b/passes/GollvmPasses.h
index b591951..0eef28c 100644
--- a/passes/GollvmPasses.h
+++ b/passes/GollvmPasses.h
@@ -23,11 +23,13 @@
void initializeGoAnnotationPass(PassRegistry&);
void initializeGoNilChecksPass(PassRegistry&);
void initializeGoStatepointsLegacyPassPass(PassRegistry&);
+void initializeGoWrappersPass(PassRegistry&);
void initializeRemoveAddrSpacePassPass(PassRegistry&);
FunctionPass *createGoAnnotationPass();
FunctionPass *createGoNilChecksPass();
ModulePass *createGoStatepointsLegacyPass();
+FunctionPass *createGoWrappersPass();
ModulePass *createRemoveAddrSpacePass(const DataLayout&);
void linkGoGC();
