go/wb.cc

// wb.cc -- Add write barriers as needed.

// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

#include "go-system.h"

#include "go-c.h"
#include "go-diagnostics.h"
#include "operator.h"
#include "lex.h"
#include "types.h"
#include "expressions.h"
#include "statements.h"
#include "runtime.h"
#include "gogo.h"

// Mark variables whose addresses are taken.  This has to be done
// before the write barrier pass and after the escape analysis pass.
// It would be nice to do this elsewhere but there isn't an obvious
// place.

class Mark_address_taken : public Traverse
{
 public:
  Mark_address_taken(Gogo* gogo)
    : Traverse(traverse_expressions),
      gogo_(gogo)
  { }

  int
  expression(Expression**);

 private:
  Gogo* gogo_;
};

// Mark variable addresses taken.

int
Mark_address_taken::expression(Expression** pexpr)
{
  Expression* expr = *pexpr;
  Unary_expression* ue = expr->unary_expression();
  if (ue != NULL)
    ue->check_operand_address_taken(this->gogo_);

  Array_index_expression* aie = expr->array_index_expression();
  if (aie != NULL
      && aie->end() != NULL
      && !aie->array()->type()->is_slice_type())
    {
      // Slice of an array. The escape analysis models this with
      // a child Node representing the address of the array.
      bool escapes = false;
      Node* n = Node::make_node(expr);
      if (n->child() == NULL
          || (n->child()->encoding() & ESCAPE_MASK) != Node::ESCAPE_NONE)
        escapes = true;
      aie->array()->address_taken(escapes);
    }

  if (expr->allocation_expression() != NULL)
    {
      Node* n = Node::make_node(expr);
      if ((n->encoding() & ESCAPE_MASK) == Node::ESCAPE_NONE)
        expr->allocation_expression()->set_allocate_on_stack();
    }
  if (expr->heap_expression() != NULL)
    {
      Node* n = Node::make_node(expr);
      if ((n->encoding() & ESCAPE_MASK) == Node::ESCAPE_NONE)
        expr->heap_expression()->set_allocate_on_stack();
    }
  if (expr->slice_literal() != NULL)
    {
      Node* n = Node::make_node(expr);
      if ((n->encoding() & ESCAPE_MASK) == Node::ESCAPE_NONE)
        expr->slice_literal()->set_storage_does_not_escape();
    }

  // Rewrite non-escaping makeslice with constant size to stack allocation.
  Unsafe_type_conversion_expression* uce =
    expr->unsafe_conversion_expression();
  if (uce != NULL
      && uce->type()->is_slice_type()
      && Node::make_node(uce->expr())->encoding() == Node::ESCAPE_NONE
      && uce->expr()->call_expression() != NULL)
    {
      Call_expression* call = uce->expr()->call_expression();
      if (call->fn()->func_expression() != NULL
          && call->fn()->func_expression()->runtime_code() == Runtime::MAKESLICE)
        {
          Expression* len_arg = call->args()->at(1);
          Expression* cap_arg = call->args()->at(2);
          Numeric_constant nclen;
          Numeric_constant nccap;
          unsigned long vlen;
          unsigned long vcap;
          if (len_arg->numeric_constant_value(&nclen)
              && cap_arg->numeric_constant_value(&nccap)
              && nclen.to_unsigned_long(&vlen) == Numeric_constant::NC_UL_VALID
              && nccap.to_unsigned_long(&vcap) == Numeric_constant::NC_UL_VALID)
            {
              // Turn it into a slice expression of an addressable array,
              // which is allocated on stack.
              Location loc = expr->location();
              Type* elmt_type = expr->type()->array_type()->element_type();
              Expression* len_expr =
                Expression::make_integer_ul(vcap, cap_arg->type(), loc);
              Type* array_type = Type::make_array_type(elmt_type, len_expr);
              Expression* alloc = Expression::make_allocation(array_type, loc);
              alloc->allocation_expression()->set_allocate_on_stack();
              Expression* array = Expression::make_unary(OPERATOR_MULT, alloc, loc);
              Expression* zero = Expression::make_integer_ul(0, len_arg->type(), loc);
              Expression* slice =
                Expression::make_array_index(array, zero, len_arg, cap_arg, loc);
              *pexpr = slice;
            }
        }
    }
  return TRAVERSE_CONTINUE;
}

// Check variables and closures do not escape when compiling runtime.

class Check_escape : public Traverse
{
 public:
  Check_escape(Gogo* gogo)
    : Traverse(traverse_expressions | traverse_variables),
      gogo_(gogo)
  { }

  int
  expression(Expression**);

  int
  variable(Named_object*);

 private:
  Gogo* gogo_;
};

int
Check_escape::variable(Named_object* no)
{
  if ((no->is_variable() && no->var_value()->is_in_heap())
      || (no->is_result_variable()
          && no->result_var_value()->is_in_heap()))
    go_error_at(no->location(),
                "%s escapes to heap, not allowed in runtime",
                no->message_name().c_str());
  return TRAVERSE_CONTINUE;
}

int
Check_escape::expression(Expression** pexpr)
{
  Expression* expr = *pexpr;
  Func_expression* fe = expr->func_expression();
  if (fe != NULL && fe->closure() != NULL)
    {
      Node* n = Node::make_node(expr);
      if (n->encoding() == Node::ESCAPE_HEAP)
        go_error_at(expr->location(),
                    "heap-allocated closure, not allowed in runtime");
    }
  return TRAVERSE_CONTINUE;
}

// Add write barriers to the IR.  This are required by the concurrent
// garbage collector.  A write barrier is needed for any write of a
// pointer into memory controlled by the garbage collector.  Write
// barriers are not required for writes to local variables that live
// on the stack.  Write barriers are only required when the runtime
// enables them, which can be checked using a run time check on
// runtime.writeBarrier.enabled.
//
// Essentially, for each assignment A = B, where A is or contains a
// pointer, and where A is not, or at any rate may not be, a stack
// variable, we rewrite it into
//     if runtime.writeBarrier.enabled {
//         typedmemmove(typeof(A), &A, &B)
//     } else {
//         A = B
//     }
//
// The test of runtime.writeBarrier.Enabled is implemented by treating
// the variable as a *uint32, and testing *runtime.writeBarrier != 0.
// This is compatible with the definition in the runtime package.
//
// For types that are pointer shared (pointers, maps, chans, funcs),
// we replaced the call to typedmemmove with writebarrierptr(&A, B).
// As far as the GC is concerned, all pointers are the same, so it
// doesn't need the type descriptor.
//
// There are possible optimizations that are not implemented.
//
// runtime.writeBarrier can only change when the goroutine is
// preempted, which in practice means when a call is made into the
// runtime package, so we could optimize by only testing it once
// between function calls.
//
// A slice could be handled with a call to writebarrierptr plus two
// integer moves.

// Traverse the IR adding write barriers.

class Write_barriers : public Traverse
{
 public:
  Write_barriers(Gogo* gogo)
    : Traverse(traverse_functions | traverse_variables | traverse_statements),
      gogo_(gogo), function_(NULL), statements_added_()
  { }

  int
  function(Named_object*);

  int
  variable(Named_object*);

  int
  statement(Block*, size_t* pindex, Statement*);

 private:
  // General IR.
  Gogo* gogo_;
  // Current function.
  Function* function_;
  // Statements introduced.
  Statement_inserter::Statements statements_added_;
};

// Traverse a function.  Just record it for later.

int
Write_barriers::function(Named_object* no)
{
  go_assert(this->function_ == NULL);
  this->function_ = no->func_value();
  int t = this->function_->traverse(this);
  this->function_ = NULL;

  if (t == TRAVERSE_EXIT)
    return t;
  return TRAVERSE_SKIP_COMPONENTS;
}

// Insert write barriers for a global variable: ensure that variable
// initialization is handled correctly.  This is rarely needed, since
// we currently don't enable background GC until after all global
// variables are initialized.  But we do need this if an init function
// calls runtime.GC.

int
Write_barriers::variable(Named_object* no)
{
  // We handle local variables in the variable declaration statement.
  // We only have to handle global variables here.
  if (!no->is_variable())
    return TRAVERSE_CONTINUE;
  Variable* var = no->var_value();
  if (!var->is_global())
    return TRAVERSE_CONTINUE;

  // Nothing to do if there is no initializer.
  Expression* init = var->init();
  if (init == NULL)
    return TRAVERSE_CONTINUE;

  // Nothing to do for variables that do not contain any pointers.
  if (!var->type()->has_pointer())
    return TRAVERSE_CONTINUE;

  // Nothing to do if the initializer is static.
  init = Expression::make_cast(var->type(), init, var->location());
  if (!var->has_pre_init() && init->is_static_initializer())
    return TRAVERSE_CONTINUE;

  // Nothing to do for a type that can not be in the heap, or a
  // pointer to a type that can not be in the heap.
  if (!var->type()->in_heap())
    return TRAVERSE_CONTINUE;
  if (var->type()->points_to() != NULL && !var->type()->points_to()->in_heap())
    return TRAVERSE_CONTINUE;

  // Otherwise change the initializer into a pre_init assignment
  // statement with a write barrier.

  // We can't check for a dependency of the variable on itself after
  // we make this change, because the preinit statement will always
  // depend on the variable (since it assigns to it).  So check for a
  // self-dependency now.
  this->gogo_->check_self_dep(no);

  // Replace the initializer.
  Location loc = init->location();
  Expression* ref = Expression::make_var_reference(no, loc);

  Statement_inserter inserter(this->gogo_, var, &this->statements_added_);
  Statement* s = this->gogo_->assign_with_write_barrier(NULL, NULL, &inserter,
							ref, init, loc);
  this->statements_added_.insert(s);

  var->add_preinit_statement(this->gogo_, s);
  var->clear_init();

  return TRAVERSE_CONTINUE;
}

// Insert write barriers for statements.

int
Write_barriers::statement(Block* block, size_t* pindex, Statement* s)
{
  if (this->statements_added_.find(s) != this->statements_added_.end())
    return TRAVERSE_SKIP_COMPONENTS;

  switch (s->classification())
    {
    default:
      break;

    case Statement::STATEMENT_VARIABLE_DECLARATION:
      {
	Variable_declaration_statement* vds =
	  s->variable_declaration_statement();
	Named_object* no = vds->var();
	Variable* var = no->var_value();

	// We may need to emit a write barrier for the initialization
	// of the variable.

	// Nothing to do for a variable with no initializer.
	Expression* init = var->init();
	if (init == NULL)
	  break;

	// Nothing to do if the variable is not in the heap.  Only
	// local variables get declaration statements, and local
	// variables on the stack do not require write barriers.
	if (!var->is_in_heap())
	  break;

	// Nothing to do if the variable does not contain any pointers.
	if (!var->type()->has_pointer())
	  break;

	// Nothing to do for a type that can not be in the heap, or a
	// pointer to a type that can not be in the heap.
	if (!var->type()->in_heap())
	  break;
	if (var->type()->points_to() != NULL
	    && !var->type()->points_to()->in_heap())
	  break;

	// Otherwise initialize the variable with a write barrier.

	Function* function = this->function_;
	Location loc = init->location();
	Statement_inserter inserter(block, pindex, &this->statements_added_);

	// Insert the variable declaration statement with no
	// initializer, so that the variable exists.
	var->clear_init();
	inserter.insert(s);

	// Create a statement that initializes the variable with a
	// write barrier.
	Expression* ref = Expression::make_var_reference(no, loc);
	Statement* assign = this->gogo_->assign_with_write_barrier(function,
								   block,
								   &inserter,
								   ref, init,
								   loc);
        this->statements_added_.insert(assign);

	// Replace the old variable declaration statement with the new
	// initialization.
	block->replace_statement(*pindex, assign);
      }
      break;

    case Statement::STATEMENT_ASSIGNMENT:
      {
	Assignment_statement* as = s->assignment_statement();
	Expression* lhs = as->lhs();
	Expression* rhs = as->rhs();

	// We may need to emit a write barrier for the assignment.

	if (!this->gogo_->assign_needs_write_barrier(lhs))
	  break;

	// Change the assignment to use a write barrier.
	Function* function = this->function_;
	Location loc = as->location();
	Statement_inserter inserter =
            Statement_inserter(block, pindex, &this->statements_added_);
	Statement* assign = this->gogo_->assign_with_write_barrier(function,
								   block,
								   &inserter,
								   lhs, rhs,
								   loc);
        this->statements_added_.insert(assign);
	block->replace_statement(*pindex, assign);
      }
      break;
    }

  return TRAVERSE_CONTINUE;
}

// The write barrier pass.

void
Gogo::add_write_barriers()
{
  if (saw_errors())
    return;

  Mark_address_taken mat(this);
  this->traverse(&mat);

  if (this->compiling_runtime() && this->package_name() == "runtime")
    {
      Check_escape chk(this);
      this->traverse(&chk);
    }

  Write_barriers wb(this);
  this->traverse(&wb);
}

// Return the runtime.writeBarrier variable.

Named_object*
Gogo::write_barrier_variable()
{
  static Named_object* write_barrier_var;
  if (write_barrier_var == NULL)
    {
      Location bloc = Linemap::predeclared_location();

      // We pretend that writeBarrier is a uint32, so that we do a
      // 32-bit load.  That is what the gc toolchain does.
      Type* uint32_type = Type::lookup_integer_type("uint32");
      Variable* var = new Variable(uint32_type, NULL, true, false, false,
				   bloc);

      bool add_to_globals;
      Package* package = this->add_imported_package("runtime", "_", false,
						    "runtime", "runtime",
						    bloc, &add_to_globals);
      write_barrier_var = Named_object::make_variable("writeBarrier",
						      package, var);
    }

  return write_barrier_var;
}

// Return whether an assignment that sets LHS needs a write barrier.

bool
Gogo::assign_needs_write_barrier(Expression* lhs)
{
  // Nothing to do if the variable does not contain any pointers.
  if (!lhs->type()->has_pointer())
    return false;

  // An assignment to a field is handled like an assignment to the
  // struct.
  while (true)
    {
      // Nothing to do for a type that can not be in the heap, or a
      // pointer to a type that can not be in the heap.  We check this
      // at each level of a struct.
      if (!lhs->type()->in_heap())
	return false;
      if (lhs->type()->points_to() != NULL
	  && !lhs->type()->points_to()->in_heap())
	return false;

      Field_reference_expression* fre = lhs->field_reference_expression();
      if (fre == NULL)
	break;
      lhs = fre->expr();
    }

  // Nothing to do for an assignment to a temporary.
  if (lhs->temporary_reference_expression() != NULL)
    return false;

  // Nothing to do for an assignment to a sink.
  if (lhs->is_sink_expression())
    return false;

  // Nothing to do for an assignment to a local variable that is not
  // on the heap.
  Var_expression* ve = lhs->var_expression();
  if (ve != NULL)
    {
      Named_object* no = ve->named_object();
      if (no->is_variable())
	{
	  Variable* var = no->var_value();
	  if (!var->is_global() && !var->is_in_heap())
	    return false;
	}
      else if (no->is_result_variable())
	{
	  Result_variable* rvar = no->result_var_value();
	  if (!rvar->is_in_heap())
	    return false;
	}
    }

  // For a struct assignment, we don't need a write barrier if all the
  // pointer types can not be in the heap.
  Struct_type* st = lhs->type()->struct_type();
  if (st != NULL)
    {
      bool in_heap = false;
      const Struct_field_list* fields = st->fields();
      for (Struct_field_list::const_iterator p = fields->begin();
	   p != fields->end();
	   p++)
	{
	  Type* ft = p->type();
	  if (!ft->has_pointer())
	    continue;
	  if (!ft->in_heap())
	    continue;
	  if (ft->points_to() != NULL && !ft->points_to()->in_heap())
	    continue;
	  in_heap = true;
	  break;
	}
      if (!in_heap)
	return false;
    }

  // Write barrier needed in other cases.
  return true;
}

// Return a statement that sets LHS to RHS using a write barrier.
// ENCLOSING is the enclosing block.

Statement*
Gogo::assign_with_write_barrier(Function* function, Block* enclosing,
				Statement_inserter* inserter, Expression* lhs,
				Expression* rhs, Location loc)
{
  if (function != NULL
      && ((function->pragmas() & GOPRAGMA_NOWRITEBARRIER) != 0
	  || (function->pragmas() & GOPRAGMA_NOWRITEBARRIERREC) != 0))
    go_error_at(loc, "write barrier prohibited");

  Type* type = lhs->type();
  go_assert(type->has_pointer());

  Expression* addr;
  if (lhs->unary_expression() != NULL
      && lhs->unary_expression()->op() == OPERATOR_MULT)
    addr = lhs->unary_expression()->operand();
  else
    {
      addr = Expression::make_unary(OPERATOR_AND, lhs, loc);
      addr->unary_expression()->set_does_not_escape();
    }
  Temporary_statement* lhs_temp = Statement::make_temporary(NULL, addr, loc);
  inserter->insert(lhs_temp);
  lhs = Expression::make_temporary_reference(lhs_temp, loc);

  if (!Type::are_identical(type, rhs->type(), false, NULL)
      && rhs->type()->interface_type() != NULL
      && !rhs->is_variable())
    {
      // May need a temporary for interface conversion.
      Temporary_statement* temp = Statement::make_temporary(NULL, rhs, loc);
      inserter->insert(temp);
      rhs = Expression::make_temporary_reference(temp, loc);
    }
  rhs = Expression::convert_for_assignment(this, type, rhs, loc);
  Temporary_statement* rhs_temp = NULL;
  if (!rhs->is_variable() && !rhs->is_constant())
    {
      rhs_temp = Statement::make_temporary(NULL, rhs, loc);
      inserter->insert(rhs_temp);
      rhs = Expression::make_temporary_reference(rhs_temp, loc);
    }

  Expression* indir =
      Expression::make_dereference(lhs, Expression::NIL_CHECK_DEFAULT, loc);
  Statement* assign = Statement::make_assignment(indir, rhs, loc);

  lhs = Expression::make_temporary_reference(lhs_temp, loc);
  if (rhs_temp != NULL)
    rhs = Expression::make_temporary_reference(rhs_temp, loc);

  Type* unsafe_ptr_type = Type::make_pointer_type(Type::make_void_type());
  lhs = Expression::make_unsafe_cast(unsafe_ptr_type, lhs, loc);

  Expression* call;
  switch (type->base()->classification())
    {
    default:
      go_unreachable();

    case Type::TYPE_ERROR:
      return assign;

    case Type::TYPE_POINTER:
    case Type::TYPE_FUNCTION:
    case Type::TYPE_MAP:
    case Type::TYPE_CHANNEL:
      // These types are all represented by a single pointer.
      call = Runtime::make_call(Runtime::WRITEBARRIERPTR, loc, 2, lhs, rhs);
      break;

    case Type::TYPE_STRING:
    case Type::TYPE_STRUCT:
    case Type::TYPE_ARRAY:
    case Type::TYPE_INTERFACE:
      {
	rhs = Expression::make_unary(OPERATOR_AND, rhs, loc);
	rhs->unary_expression()->set_does_not_escape();
	call = Runtime::make_call(Runtime::TYPEDMEMMOVE, loc, 3,
				  Expression::make_type_descriptor(type, loc),
				  lhs, rhs);
      }
      break;
    }

  return this->check_write_barrier(enclosing, assign,
				   Statement::make_statement(call, false));
}

// Return a statement that tests whether write barriers are enabled
// and executes either the efficient code or the write barrier
// function call, depending.

Statement*
Gogo::check_write_barrier(Block* enclosing, Statement* without,
			  Statement* with)
{
  Location loc = without->location();
  Named_object* wb = this->write_barrier_variable();
  Expression* ref = Expression::make_var_reference(wb, loc);
  Expression* zero = Expression::make_integer_ul(0, ref->type(), loc);
  Expression* cond = Expression::make_binary(OPERATOR_EQEQ, ref, zero, loc);

  Block* then_block = new Block(enclosing, loc);
  then_block->add_statement(without);

  Block* else_block = new Block(enclosing, loc);
  else_block->add_statement(with);

  return Statement::make_if_statement(cond, then_block, else_block, loc);
}