compiler: intrinsify runtime/internal/atomic functions
Currently runtime/internal/atomic functions are implemented in C
using C compiler intrinsics. This CL lets the Go frontend
recognize these functions and turn them into intrinsics directly.
Corresponding change to the GCC backend:
Index: gcc/go/go-gcc.cc
===================================================================
--- gcc/go/go-gcc.cc (revision 271105)
+++ gcc/go/go-gcc.cc (working copy)
@@ -776,6 +776,109 @@
this->define_builtin(BUILT_IN_UNREACHABLE, "__builtin_unreachable", NULL,
build_function_type(void_type_node, void_list_node),
true, true);
+
+ // We provide some atomic functions.
+ t = build_function_type_list(uint32_type_node,
+ ptr_type_node,
+ integer_type_node,
+ NULL_TREE);
+ this->define_builtin(BUILT_IN_ATOMIC_LOAD_4, "__atomic_load_4", NULL,
+ t, false, false);
+
+ t = build_function_type_list(uint64_type_node,
+ ptr_type_node,
+ integer_type_node,
+ NULL_TREE);
+ this->define_builtin(BUILT_IN_ATOMIC_LOAD_8, "__atomic_load_8", NULL,
+ t, false, false);
+
+ t = build_function_type_list(void_type_node,
+ ptr_type_node,
+ uint32_type_node,
+ integer_type_node,
+ NULL_TREE);
+ this->define_builtin(BUILT_IN_ATOMIC_STORE_4, "__atomic_store_4", NULL,
+ t, false, false);
+
+ t = build_function_type_list(void_type_node,
+ ptr_type_node,
+ uint64_type_node,
+ integer_type_node,
+ NULL_TREE);
+ this->define_builtin(BUILT_IN_ATOMIC_STORE_8, "__atomic_store_8", NULL,
+ t, false, false);
+
+ t = build_function_type_list(uint32_type_node,
+ ptr_type_node,
+ uint32_type_node,
+ integer_type_node,
+ NULL_TREE);
+ this->define_builtin(BUILT_IN_ATOMIC_EXCHANGE_4, "__atomic_exchange_4", NULL,
+ t, false, false);
+
+ t = build_function_type_list(uint64_type_node,
+ ptr_type_node,
+ uint64_type_node,
+ integer_type_node,
+ NULL_TREE);
+ this->define_builtin(BUILT_IN_ATOMIC_EXCHANGE_8, "__atomic_exchange_8", NULL,
+ t, false, false);
+
+ t = build_function_type_list(boolean_type_node,
+ ptr_type_node,
+ ptr_type_node,
+ uint32_type_node,
+ boolean_type_node,
+ integer_type_node,
+ integer_type_node,
+ NULL_TREE);
+ this->define_builtin(BUILT_IN_ATOMIC_COMPARE_EXCHANGE_4,
+ "__atomic_compare_exchange_4", NULL,
+ t, false, false);
+
+ t = build_function_type_list(boolean_type_node,
+ ptr_type_node,
+ ptr_type_node,
+ uint64_type_node,
+ boolean_type_node,
+ integer_type_node,
+ integer_type_node,
+ NULL_TREE);
+ this->define_builtin(BUILT_IN_ATOMIC_COMPARE_EXCHANGE_8,
+ "__atomic_compare_exchange_8", NULL,
+ t, false, false);
+
+ t = build_function_type_list(uint32_type_node,
+ ptr_type_node,
+ uint32_type_node,
+ integer_type_node,
+ NULL_TREE);
+ this->define_builtin(BUILT_IN_ATOMIC_ADD_FETCH_4, "__atomic_add_fetch_4", NULL,
+ t, false, false);
+
+ t = build_function_type_list(uint64_type_node,
+ ptr_type_node,
+ uint64_type_node,
+ integer_type_node,
+ NULL_TREE);
+ this->define_builtin(BUILT_IN_ATOMIC_ADD_FETCH_8, "__atomic_add_fetch_8", NULL,
+ t, false, false);
+
+ t = build_function_type_list(unsigned_char_type_node,
+ ptr_type_node,
+ unsigned_char_type_node,
+ integer_type_node,
+ NULL_TREE);
+ this->define_builtin(BUILT_IN_ATOMIC_AND_FETCH_1, "__atomic_and_fetch_1", NULL,
+ t, false, false);
+
+ t = build_function_type_list(unsigned_char_type_node,
+ ptr_type_node,
+ unsigned_char_type_node,
+ integer_type_node,
+ NULL_TREE);
+ this->define_builtin(BUILT_IN_ATOMIC_OR_FETCH_1, "__atomic_or_fetch_1", NULL,
+ t, false, false);
}
// Get an unnamed integer type.
Change-Id: Ibdc77bac3a9f7890d468fe5e244ed4d452741df8
Reviewed-on: https://go-review.googlesource.com/c/gofrontend/+/176918
Reviewed-by: Ian Lance Taylor <iant@golang.org>
diff --git a/go/expressions.cc b/go/expressions.cc
index d162b05..dd559b4 100644
--- a/go/expressions.cc
+++ b/go/expressions.cc
@@ -10460,9 +10460,16 @@
Location loc = this->location();
Type* int_type = Type::lookup_integer_type("int");
+ Type* int32_type = Type::lookup_integer_type("int32");
+ Type* int64_type = Type::lookup_integer_type("int64");
+ Type* uint_type = Type::lookup_integer_type("uint");
Type* uint32_type = Type::lookup_integer_type("uint32");
Type* uint64_type = Type::lookup_integer_type("uint64");
Type* uintptr_type = Type::lookup_integer_type("uintptr");
+ Type* pointer_type = Type::make_pointer_type(Type::make_void_type());
+
+ int int_size = int_type->named_type()->real_type()->integer_type()->bits() / 8;
+ int ptr_size = uintptr_type->named_type()->real_type()->integer_type()->bits() / 8;
if (package == "runtime")
{
@@ -10545,6 +10552,242 @@
return Expression::make_conditional(cmp, c64, call, loc);
}
}
+ else if (package == "runtime/internal/atomic")
+ {
+ int memorder = __ATOMIC_SEQ_CST;
+
+ if ((name == "Load" || name == "Load64" || name == "Loadint64" || name == "Loadp"
+ || name == "Loaduint" || name == "Loaduintptr" || name == "LoadAcq")
+ && this->args_ != NULL && this->args_->size() == 1)
+ {
+ if (int_size < 8 && (name == "Load64" || name == "Loadint64"))
+ // On 32-bit architectures we need to check alignment.
+ // Not intrinsify for now.
+ return NULL;
+
+ Runtime::Function code;
+ Type* res_type;
+ if (name == "Load")
+ {
+ code = Runtime::ATOMIC_LOAD_4;
+ res_type = uint32_type;
+ }
+ else if (name == "Load64")
+ {
+ code = Runtime::ATOMIC_LOAD_8;
+ res_type = uint64_type;
+ }
+ else if (name == "Loadint64")
+ {
+ code = Runtime::ATOMIC_LOAD_8;
+ res_type = int64_type;
+ }
+ else if (name == "Loaduint")
+ {
+ code = (int_size == 8
+ ? Runtime::ATOMIC_LOAD_8
+ : Runtime::ATOMIC_LOAD_4);
+ res_type = uint_type;
+ }
+ else if (name == "Loaduintptr")
+ {
+ code = (ptr_size == 8
+ ? Runtime::ATOMIC_LOAD_8
+ : Runtime::ATOMIC_LOAD_4);
+ res_type = uintptr_type;
+ }
+ else if (name == "Loadp")
+ {
+ code = (ptr_size == 8
+ ? Runtime::ATOMIC_LOAD_8
+ : Runtime::ATOMIC_LOAD_4);
+ res_type = pointer_type;
+ }
+ else if (name == "LoadAcq")
+ {
+ code = Runtime::ATOMIC_LOAD_4;
+ res_type = uint32_type;
+ memorder = __ATOMIC_ACQUIRE;
+ }
+ else
+ go_unreachable();
+ Expression* a1 = this->args_->front();
+ Expression* a2 = Expression::make_integer_ul(memorder, int32_type, loc);
+ Expression* call = Runtime::make_call(code, loc, 2, a1, a2);
+ return Expression::make_unsafe_cast(res_type, call, loc);
+ }
+
+ if ((name == "Store" || name == "Store64" || name == "StorepNoWB"
+ || name == "Storeuintptr" || name == "StoreRel")
+ && this->args_ != NULL && this->args_->size() == 2)
+ {
+ if (int_size < 8 && name == "Store64")
+ return NULL;
+
+ Runtime::Function code;
+ Expression* a1 = this->args_->at(0);
+ Expression* a2 = this->args_->at(1);
+ if (name == "Store")
+ code = Runtime::ATOMIC_STORE_4;
+ else if (name == "Store64")
+ code = Runtime::ATOMIC_STORE_8;
+ else if (name == "Storeuintptr")
+ code = (ptr_size == 8 ? Runtime::ATOMIC_STORE_8 : Runtime::ATOMIC_STORE_4);
+ else if (name == "StorepNoWB")
+ {
+ code = (ptr_size == 8 ? Runtime::ATOMIC_STORE_8 : Runtime::ATOMIC_STORE_4);
+ a2 = Expression::make_unsafe_cast(uintptr_type, a2, loc);
+ a2 = Expression::make_cast(uint64_type, a2, loc);
+ }
+ else if (name == "StoreRel")
+ {
+ code = Runtime::ATOMIC_STORE_4;
+ memorder = __ATOMIC_RELEASE;
+ }
+ else
+ go_unreachable();
+ Expression* a3 = Expression::make_integer_ul(memorder, int32_type, loc);
+ return Runtime::make_call(code, loc, 3, a1, a2, a3);
+ }
+
+ if ((name == "Xchg" || name == "Xchg64" || name == "Xchguintptr")
+ && this->args_ != NULL && this->args_->size() == 2)
+ {
+ if (int_size < 8 && name == "Xchg64")
+ return NULL;
+
+ Runtime::Function code;
+ Type* res_type;
+ if (name == "Xchg")
+ {
+ code = Runtime::ATOMIC_EXCHANGE_4;
+ res_type = uint32_type;
+ }
+ else if (name == "Xchg64")
+ {
+ code = Runtime::ATOMIC_EXCHANGE_8;
+ res_type = uint64_type;
+ }
+ else if (name == "Xchguintptr")
+ {
+ code = (ptr_size == 8
+ ? Runtime::ATOMIC_EXCHANGE_8
+ : Runtime::ATOMIC_EXCHANGE_4);
+ res_type = uintptr_type;
+ }
+ else
+ go_unreachable();
+ Expression* a1 = this->args_->at(0);
+ Expression* a2 = this->args_->at(1);
+ Expression* a3 = Expression::make_integer_ul(memorder, int32_type, loc);
+ Expression* call = Runtime::make_call(code, loc, 3, a1, a2, a3);
+ return Expression::make_cast(res_type, call, loc);
+ }
+
+ if ((name == "Cas" || name == "Cas64" || name == "Casuintptr"
+ || name == "Casp1" || name == "CasRel")
+ && this->args_ != NULL && this->args_->size() == 3)
+ {
+ if (int_size < 8 && name == "Cas64")
+ return NULL;
+
+ Runtime::Function code;
+ Expression* a1 = this->args_->at(0);
+
+ // Builtin cas takes a pointer to the old value.
+ // Store it in a temporary and take the address.
+ Expression* a2 = this->args_->at(1);
+ Temporary_statement* ts = Statement::make_temporary(NULL, a2, loc);
+ inserter->insert(ts);
+ a2 = Expression::make_temporary_reference(ts, loc);
+ a2 = Expression::make_unary(OPERATOR_AND, a2, loc);
+
+ Expression* a3 = this->args_->at(2);
+ if (name == "Cas")
+ code = Runtime::ATOMIC_COMPARE_EXCHANGE_4;
+ else if (name == "Cas64")
+ code = Runtime::ATOMIC_COMPARE_EXCHANGE_8;
+ else if (name == "Casuintptr")
+ code = (ptr_size == 8
+ ? Runtime::ATOMIC_COMPARE_EXCHANGE_8
+ : Runtime::ATOMIC_COMPARE_EXCHANGE_4);
+ else if (name == "Casp1")
+ {
+ code = (ptr_size == 8
+ ? Runtime::ATOMIC_COMPARE_EXCHANGE_8
+ : Runtime::ATOMIC_COMPARE_EXCHANGE_4);
+ a3 = Expression::make_unsafe_cast(uintptr_type, a3, loc);
+ a3 = Expression::make_cast(uint64_type, a3, loc);
+ }
+ else if (name == "CasRel")
+ {
+ code = Runtime::ATOMIC_COMPARE_EXCHANGE_4;
+ memorder = __ATOMIC_RELEASE;
+ }
+ else
+ go_unreachable();
+ Expression* a4 = Expression::make_boolean(false, loc);
+ Expression* a5 = Expression::make_integer_ul(memorder, int32_type, loc);
+ Expression* a6 = Expression::make_integer_ul(__ATOMIC_RELAXED, int32_type, loc);
+ return Runtime::make_call(code, loc, 6, a1, a2, a3, a4, a5, a6);
+ }
+
+ if ((name == "Xadd" || name == "Xadd64" || name == "Xaddint64"
+ || name == "Xadduintptr")
+ && this->args_ != NULL && this->args_->size() == 2)
+ {
+ if (int_size < 8 && (name == "Xadd64" || name == "Xaddint64"))
+ return NULL;
+
+ Runtime::Function code;
+ Type* res_type;
+ if (name == "Xadd")
+ {
+ code = Runtime::ATOMIC_ADD_FETCH_4;
+ res_type = uint32_type;
+ }
+ else if (name == "Xadd64")
+ {
+ code = Runtime::ATOMIC_ADD_FETCH_8;
+ res_type = uint64_type;
+ }
+ else if (name == "Xaddint64")
+ {
+ code = Runtime::ATOMIC_ADD_FETCH_8;
+ res_type = int64_type;
+ }
+ else if (name == "Xadduintptr")
+ {
+ code = (ptr_size == 8
+ ? Runtime::ATOMIC_ADD_FETCH_8
+ : Runtime::ATOMIC_ADD_FETCH_4);
+ res_type = uintptr_type;
+ }
+ else
+ go_unreachable();
+ Expression* a1 = this->args_->at(0);
+ Expression* a2 = this->args_->at(1);
+ Expression* a3 = Expression::make_integer_ul(memorder, int32_type, loc);
+ Expression* call = Runtime::make_call(code, loc, 3, a1, a2, a3);
+ return Expression::make_cast(res_type, call, loc);
+ }
+
+ if ((name == "And8" || name == "Or8")
+ && this->args_ != NULL && this->args_->size() == 2)
+ {
+ Runtime::Function code;
+ if (name == "And8")
+ code = Runtime::ATOMIC_AND_FETCH_1;
+ else if (name == "Or8")
+ code = Runtime::ATOMIC_OR_FETCH_1;
+ else
+ go_unreachable();
+ Expression* a1 = this->args_->at(0);
+ Expression* a2 = this->args_->at(1);
+ Expression* a3 = Expression::make_integer_ul(memorder, int32_type, loc);
+ return Runtime::make_call(code, loc, 3, a1, a2, a3);
+ }
+ }
return NULL;
}
diff --git a/go/runtime.cc b/go/runtime.cc
index 289d6bf..28aca44 100644
--- a/go/runtime.cc
+++ b/go/runtime.cc
@@ -30,6 +30,8 @@
RFT_BOOLPTR,
// Go type int, C type intgo.
RFT_INT,
+ // Go type uint8, C type uint8_t.
+ RFT_UINT8,
// Go type int32, C type int32_t.
RFT_INT32,
// Go type uint32, C type uint32_t.
@@ -109,6 +111,10 @@
t = Type::lookup_integer_type("int");
break;
+ case RFT_UINT8:
+ t = Type::lookup_integer_type("uint8");
+ break;
+
case RFT_INT32:
t = Type::lookup_integer_type("int32");
break;
@@ -250,6 +256,7 @@
case RFT_BOOL:
case RFT_BOOLPTR:
case RFT_INT:
+ case RFT_UINT8:
case RFT_INT32:
case RFT_UINT32:
case RFT_INT64:
diff --git a/go/runtime.def b/go/runtime.def
index a5264ee..226eeac 100644
--- a/go/runtime.def
+++ b/go/runtime.def
@@ -396,6 +396,38 @@
DEF_GO_RUNTIME(BUILTIN_CTZ, "__builtin_ctz", P1(UINT32), R1(INT32))
DEF_GO_RUNTIME(BUILTIN_CTZLL, "__builtin_ctzll", P1(UINT64), R1(INT32))
+// Atomics.
+DEF_GO_RUNTIME(ATOMIC_LOAD_4, "__atomic_load_4", P2(POINTER, INT32),
+ R1(UINT32))
+DEF_GO_RUNTIME(ATOMIC_LOAD_8, "__atomic_load_8", P2(POINTER, INT32),
+ R1(UINT64))
+DEF_GO_RUNTIME(ATOMIC_STORE_4, "__atomic_store_4", P3(POINTER, UINT32, INT32),
+ R0())
+DEF_GO_RUNTIME(ATOMIC_STORE_8, "__atomic_store_8", P3(POINTER, UINT64, INT32),
+ R0())
+DEF_GO_RUNTIME(ATOMIC_EXCHANGE_4, "__atomic_exchange_4", P3(POINTER, UINT32, INT32),
+ R1(UINT32))
+DEF_GO_RUNTIME(ATOMIC_EXCHANGE_8, "__atomic_exchange_8", P3(POINTER, UINT64, INT32),
+ R1(UINT64))
+DEF_GO_RUNTIME(ATOMIC_COMPARE_EXCHANGE_4, "__atomic_compare_exchange_4",
+ P6(POINTER, POINTER, UINT32, BOOL, INT32, INT32),
+ R1(BOOL))
+DEF_GO_RUNTIME(ATOMIC_COMPARE_EXCHANGE_8, "__atomic_compare_exchange_8",
+ P6(POINTER, POINTER, UINT64, BOOL, INT32, INT32),
+ R1(BOOL))
+DEF_GO_RUNTIME(ATOMIC_ADD_FETCH_4, "__atomic_add_fetch_4",
+ P3(POINTER, UINT32, INT32),
+ R1(UINT32))
+DEF_GO_RUNTIME(ATOMIC_ADD_FETCH_8, "__atomic_add_fetch_8",
+ P3(POINTER, UINT64, INT32),
+ R1(UINT64))
+DEF_GO_RUNTIME(ATOMIC_AND_FETCH_1, "__atomic_and_fetch_1",
+ P3(POINTER, UINT8, INT32),
+ R1(UINT8))
+DEF_GO_RUNTIME(ATOMIC_OR_FETCH_1, "__atomic_or_fetch_1",
+ P3(POINTER, UINT8, INT32),
+ R1(UINT8))
+
// Remove helper macros.
#undef ABFT6
#undef ABFT2
