runtime: prevent deadlock when profiling signal arrives during traceback

Traceback routines, e.g. callers and funcentry, may call
__go_get_backtrace_state. If a profiling signal arrives while we
are in the critical section of __go_get_backtrace_state, it tries
to do a traceback, which also calls __go_get_backtrace_state,
which tries to enter the same critical section and will deadlock.
Prevent this deadlock by setting up runtime_in_callers before
calling __go_get_backtrace_state.

Found while investigating golang/go#29448. Will add a test in the
next CL.

Updates golang/go#29448.

Change-Id: Ifb50c50aedc6faf7259757b82e11b30b867bc70d
Reviewed-on: https://go-review.googlesource.com/c/156037
Reviewed-by: Ian Lance Taylor <iant@golang.org>
diff --git a/libgo/runtime/go-caller.c b/libgo/runtime/go-caller.c
index 9d22f9f..6fe4340 100644
--- a/libgo/runtime/go-caller.c
+++ b/libgo/runtime/go-caller.c
@@ -137,7 +137,9 @@
 
   runtime_memclr (&c, sizeof c);
   c.index = index;
+  runtime_xadd (&__go_runtime_in_callers, 1);
   state = __go_get_backtrace_state ();
+  runtime_xadd (&__go_runtime_in_callers, -1);
   backtrace_pcinfo (state, pc, callback, error_callback, &c);
   *fn = c.fn;
   *file = c.file;
@@ -169,8 +171,13 @@
 static _Bool
 __go_symbol_value (uintptr pc, uintptr *val)
 {
+  struct backtrace_state *state;
+
   *val = 0;
-  backtrace_syminfo (__go_get_backtrace_state (), pc, syminfo_callback,
+  runtime_xadd (&__go_runtime_in_callers, 1);
+  state = __go_get_backtrace_state ();
+  runtime_xadd (&__go_runtime_in_callers, -1);
+  backtrace_syminfo (state, pc, syminfo_callback,
 		     error_callback, val);
   return *val != 0;
 }
diff --git a/libgo/runtime/go-callers.c b/libgo/runtime/go-callers.c
index 78ada7a..7ea7085 100644
--- a/libgo/runtime/go-callers.c
+++ b/libgo/runtime/go-callers.c
@@ -202,8 +202,8 @@
   data.index = 0;
   data.max = m;
   data.keep_thunks = keep_thunks;
-  state = __go_get_backtrace_state ();
   runtime_xadd (&__go_runtime_in_callers, 1);
+  state = __go_get_backtrace_state ();
   backtrace_full (state, 0, callback, error_callback, &data);
   runtime_xadd (&__go_runtime_in_callers, -1);