Server : Apache/2.4.6 (CentOS) OpenSSL/1.0.2k-fips PHP/7.4.20 System : Linux st2.domain.com 3.10.0-1127.10.1.el7.x86_64 #1 SMP Wed Jun 3 14:28:03 UTC 2020 x86_64 User : apache ( 48) PHP Version : 7.4.20 Disable Function : NONE Directory : /home/real/node-v13.0.1/deps/v8/src/wasm/baseline/ |
// Copyright 2017 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "src/wasm/baseline/liftoff-assembler.h"
#include <sstream>
#include "src/base/optional.h"
#include "src/codegen/assembler-inl.h"
#include "src/codegen/macro-assembler-inl.h"
#include "src/compiler/linkage.h"
#include "src/compiler/wasm-compiler.h"
#include "src/utils/ostreams.h"
#include "src/wasm/function-body-decoder-impl.h"
#include "src/wasm/wasm-linkage.h"
#include "src/wasm/wasm-opcodes.h"
namespace v8 {
namespace internal {
namespace wasm {
using VarState = LiftoffAssembler::VarState;
namespace {
#define __ asm_->
#define TRACE(...) \
do { \
if (FLAG_trace_liftoff) PrintF("[liftoff] " __VA_ARGS__); \
} while (false)
class StackTransferRecipe {
struct RegisterMove {
LiftoffRegister src;
ValueType type;
constexpr RegisterMove(LiftoffRegister src, ValueType type)
: src(src), type(type) {}
};
struct RegisterLoad {
enum LoadKind : uint8_t {
kConstant, // load a constant value into a register.
kStack, // fill a register from a stack slot.
kLowHalfStack, // fill a register from the low half of a stack slot.
kHighHalfStack // fill a register from the high half of a stack slot.
};
LoadKind kind;
ValueType type;
int32_t value; // i32 constant value or stack index, depending on kind.
// Named constructors.
static RegisterLoad Const(WasmValue constant) {
if (constant.type() == kWasmI32) {
return {kConstant, kWasmI32, constant.to_i32()};
}
DCHECK_EQ(kWasmI64, constant.type());
DCHECK_EQ(constant.to_i32_unchecked(), constant.to_i64_unchecked());
return {kConstant, kWasmI64, constant.to_i32_unchecked()};
}
static RegisterLoad Stack(int32_t stack_index, ValueType type) {
return {kStack, type, stack_index};
}
static RegisterLoad HalfStack(int32_t stack_index, RegPairHalf half) {
return {half == kLowWord ? kLowHalfStack : kHighHalfStack, kWasmI32,
stack_index};
}
private:
RegisterLoad(LoadKind kind, ValueType type, int32_t value)
: kind(kind), type(type), value(value) {}
};
public:
explicit StackTransferRecipe(LiftoffAssembler* wasm_asm) : asm_(wasm_asm) {}
~StackTransferRecipe() { Execute(); }
void Execute() {
// First, execute register moves. Then load constants and stack values into
// registers.
ExecuteMoves();
DCHECK(move_dst_regs_.is_empty());
ExecuteLoads();
DCHECK(load_dst_regs_.is_empty());
}
void TransferStackSlot(const LiftoffAssembler::CacheState& dst_state,
uint32_t dst_index,
const LiftoffAssembler::CacheState& src_state,
uint32_t src_index) {
const VarState& dst = dst_state.stack_state[dst_index];
const VarState& src = src_state.stack_state[src_index];
DCHECK_EQ(dst.type(), src.type());
switch (dst.loc()) {
case VarState::kStack:
switch (src.loc()) {
case VarState::kStack:
if (src_index == dst_index) break;
asm_->MoveStackValue(dst_index, src_index, src.type());
break;
case VarState::kRegister:
asm_->Spill(dst_index, src.reg(), src.type());
break;
case VarState::kIntConst:
asm_->Spill(dst_index, src.constant());
break;
}
break;
case VarState::kRegister:
LoadIntoRegister(dst.reg(), src, src_index);
break;
case VarState::kIntConst:
DCHECK_EQ(dst, src);
break;
}
}
void LoadIntoRegister(LiftoffRegister dst,
const LiftoffAssembler::VarState& src,
uint32_t src_index) {
switch (src.loc()) {
case VarState::kStack:
LoadStackSlot(dst, src_index, src.type());
break;
case VarState::kRegister:
DCHECK_EQ(dst.reg_class(), src.reg_class());
if (dst != src.reg()) MoveRegister(dst, src.reg(), src.type());
break;
case VarState::kIntConst:
LoadConstant(dst, src.constant());
break;
}
}
void LoadI64HalfIntoRegister(LiftoffRegister dst,
const LiftoffAssembler::VarState& src,
uint32_t index, RegPairHalf half) {
// Use CHECK such that the remaining code is statically dead if
// {kNeedI64RegPair} is false.
CHECK(kNeedI64RegPair);
DCHECK_EQ(kWasmI64, src.type());
switch (src.loc()) {
case VarState::kStack:
LoadI64HalfStackSlot(dst, index, half);
break;
case VarState::kRegister: {
LiftoffRegister src_half =
half == kLowWord ? src.reg().low() : src.reg().high();
if (dst != src_half) MoveRegister(dst, src_half, kWasmI32);
break;
}
case VarState::kIntConst:
int32_t value = src.i32_const();
// The high word is the sign extension of the low word.
if (half == kHighWord) value = value >> 31;
LoadConstant(dst, WasmValue(value));
break;
}
}
void MoveRegister(LiftoffRegister dst, LiftoffRegister src, ValueType type) {
DCHECK_NE(dst, src);
DCHECK_EQ(dst.reg_class(), src.reg_class());
DCHECK_EQ(reg_class_for(type), src.reg_class());
if (src.is_pair()) {
DCHECK_EQ(kWasmI64, type);
if (dst.low() != src.low()) MoveRegister(dst.low(), src.low(), kWasmI32);
if (dst.high() != src.high())
MoveRegister(dst.high(), src.high(), kWasmI32);
return;
}
if (move_dst_regs_.has(dst)) {
DCHECK_EQ(register_move(dst)->src, src);
// Non-fp registers can only occur with the exact same type.
DCHECK_IMPLIES(!dst.is_fp(), register_move(dst)->type == type);
// It can happen that one fp register holds both the f32 zero and the f64
// zero, as the initial value for local variables. Move the value as f64
// in that case.
if (type == kWasmF64) register_move(dst)->type = kWasmF64;
return;
}
move_dst_regs_.set(dst);
++*src_reg_use_count(src);
*register_move(dst) = {src, type};
}
void LoadConstant(LiftoffRegister dst, WasmValue value) {
DCHECK(!load_dst_regs_.has(dst));
load_dst_regs_.set(dst);
if (dst.is_pair()) {
DCHECK_EQ(kWasmI64, value.type());
int64_t i64 = value.to_i64();
*register_load(dst.low()) =
RegisterLoad::Const(WasmValue(static_cast<int32_t>(i64)));
*register_load(dst.high()) =
RegisterLoad::Const(WasmValue(static_cast<int32_t>(i64 >> 32)));
} else {
*register_load(dst) = RegisterLoad::Const(value);
}
}
void LoadStackSlot(LiftoffRegister dst, uint32_t stack_index,
ValueType type) {
if (load_dst_regs_.has(dst)) {
// It can happen that we spilled the same register to different stack
// slots, and then we reload them later into the same dst register.
// In that case, it is enough to load one of the stack slots.
return;
}
load_dst_regs_.set(dst);
if (dst.is_pair()) {
DCHECK_EQ(kWasmI64, type);
*register_load(dst.low()) =
RegisterLoad::HalfStack(stack_index, kLowWord);
*register_load(dst.high()) =
RegisterLoad::HalfStack(stack_index, kHighWord);
} else {
*register_load(dst) = RegisterLoad::Stack(stack_index, type);
}
}
void LoadI64HalfStackSlot(LiftoffRegister dst, uint32_t stack_index,
RegPairHalf half) {
if (load_dst_regs_.has(dst)) {
// It can happen that we spilled the same register to different stack
// slots, and then we reload them later into the same dst register.
// In that case, it is enough to load one of the stack slots.
return;
}
load_dst_regs_.set(dst);
*register_load(dst) = RegisterLoad::HalfStack(stack_index, half);
}
private:
using MovesStorage =
std::aligned_storage<kAfterMaxLiftoffRegCode * sizeof(RegisterMove),
alignof(RegisterMove)>::type;
using LoadsStorage =
std::aligned_storage<kAfterMaxLiftoffRegCode * sizeof(RegisterLoad),
alignof(RegisterLoad)>::type;
ASSERT_TRIVIALLY_COPYABLE(RegisterMove);
ASSERT_TRIVIALLY_COPYABLE(RegisterLoad);
MovesStorage register_moves_; // uninitialized
LoadsStorage register_loads_; // uninitialized
int src_reg_use_count_[kAfterMaxLiftoffRegCode] = {0};
LiftoffRegList move_dst_regs_;
LiftoffRegList load_dst_regs_;
LiftoffAssembler* const asm_;
RegisterMove* register_move(LiftoffRegister reg) {
return reinterpret_cast<RegisterMove*>(®ister_moves_) +
reg.liftoff_code();
}
RegisterLoad* register_load(LiftoffRegister reg) {
return reinterpret_cast<RegisterLoad*>(®ister_loads_) +
reg.liftoff_code();
}
int* src_reg_use_count(LiftoffRegister reg) {
return src_reg_use_count_ + reg.liftoff_code();
}
void ExecuteMove(LiftoffRegister dst) {
RegisterMove* move = register_move(dst);
DCHECK_EQ(0, *src_reg_use_count(dst));
asm_->Move(dst, move->src, move->type);
ClearExecutedMove(dst);
}
void ClearExecutedMove(LiftoffRegister dst) {
DCHECK(move_dst_regs_.has(dst));
move_dst_regs_.clear(dst);
RegisterMove* move = register_move(dst);
DCHECK_LT(0, *src_reg_use_count(move->src));
if (--*src_reg_use_count(move->src)) return;
// src count dropped to zero. If this is a destination register, execute
// that move now.
if (!move_dst_regs_.has(move->src)) return;
ExecuteMove(move->src);
}
void ExecuteMoves() {
// Execute all moves whose {dst} is not being used as src in another move.
// If any src count drops to zero, also (transitively) execute the
// corresponding move to that register.
for (LiftoffRegister dst : move_dst_regs_) {
// Check if already handled via transitivity in {ClearExecutedMove}.
if (!move_dst_regs_.has(dst)) continue;
if (*src_reg_use_count(dst)) continue;
ExecuteMove(dst);
}
// All remaining moves are parts of a cycle. Just spill the first one, then
// process all remaining moves in that cycle. Repeat for all cycles.
uint32_t next_spill_slot = asm_->cache_state()->stack_height();
while (!move_dst_regs_.is_empty()) {
// TODO(clemensh): Use an unused register if available.
LiftoffRegister dst = move_dst_regs_.GetFirstRegSet();
RegisterMove* move = register_move(dst);
LiftoffRegister spill_reg = move->src;
asm_->Spill(next_spill_slot, spill_reg, move->type);
// Remember to reload into the destination register later.
LoadStackSlot(dst, next_spill_slot, move->type);
++next_spill_slot;
ClearExecutedMove(dst);
}
}
void ExecuteLoads() {
for (LiftoffRegister dst : load_dst_regs_) {
RegisterLoad* load = register_load(dst);
switch (load->kind) {
case RegisterLoad::kConstant:
asm_->LoadConstant(dst, load->type == kWasmI64
? WasmValue(int64_t{load->value})
: WasmValue(int32_t{load->value}));
break;
case RegisterLoad::kStack:
asm_->Fill(dst, load->value, load->type);
break;
case RegisterLoad::kLowHalfStack:
// Half of a register pair, {dst} must be a gp register.
asm_->FillI64Half(dst.gp(), load->value, kLowWord);
break;
case RegisterLoad::kHighHalfStack:
// Half of a register pair, {dst} must be a gp register.
asm_->FillI64Half(dst.gp(), load->value, kHighWord);
break;
}
}
load_dst_regs_ = {};
}
DISALLOW_COPY_AND_ASSIGN(StackTransferRecipe);
};
class RegisterReuseMap {
public:
void Add(LiftoffRegister src, LiftoffRegister dst) {
if (auto previous = Lookup(src)) {
DCHECK_EQ(previous, dst);
return;
}
map_.emplace_back(src);
map_.emplace_back(dst);
}
base::Optional<LiftoffRegister> Lookup(LiftoffRegister src) {
for (auto it = map_.begin(), end = map_.end(); it != end; it += 2) {
if (it->is_pair() == src.is_pair() && *it == src) return *(it + 1);
}
return {};
}
private:
// {map_} holds pairs of <src, dst>.
base::SmallVector<LiftoffRegister, 8> map_;
};
enum MergeKeepStackSlots : bool {
kKeepStackSlots = true,
kTurnStackSlotsIntoRegisters = false
};
enum MergeAllowConstants : bool {
kConstantsAllowed = true,
kConstantsNotAllowed = false
};
enum ReuseRegisters : bool {
kReuseRegisters = true,
kNoReuseRegisters = false
};
void InitMergeRegion(LiftoffAssembler::CacheState* state,
const VarState* source, VarState* target, uint32_t count,
MergeKeepStackSlots keep_stack_slots,
MergeAllowConstants allow_constants,
ReuseRegisters reuse_registers, LiftoffRegList used_regs) {
RegisterReuseMap register_reuse_map;
for (const VarState* source_end = source + count; source < source_end;
++source, ++target) {
if ((source->is_stack() && keep_stack_slots) ||
(source->is_const() && allow_constants)) {
*target = *source;
continue;
}
base::Optional<LiftoffRegister> reg;
// First try: Keep the same register, if it's free.
if (source->is_reg() && state->is_free(source->reg())) {
reg = source->reg();
}
// Second try: Use the same register we used before (if we reuse registers).
if (!reg && reuse_registers) {
reg = register_reuse_map.Lookup(source->reg());
}
// Third try: Use any free register.
RegClass rc = reg_class_for(source->type());
if (!reg && state->has_unused_register(rc, used_regs)) {
reg = state->unused_register(rc, used_regs);
}
if (!reg) {
// No free register; make this a stack slot.
*target = VarState(source->type());
continue;
}
if (reuse_registers) register_reuse_map.Add(source->reg(), *reg);
state->inc_used(*reg);
*target = VarState(source->type(), *reg);
}
}
} // namespace
// TODO(clemensh): Don't copy the full parent state (this makes us N^2).
void LiftoffAssembler::CacheState::InitMerge(const CacheState& source,
uint32_t num_locals,
uint32_t arity,
uint32_t stack_depth) {
// |------locals------|---(in between)----|--(discarded)--|----merge----|
// <-- num_locals --> <-- stack_depth -->^stack_base <-- arity -->
uint32_t stack_base = stack_depth + num_locals;
uint32_t target_height = stack_base + arity;
uint32_t discarded = source.stack_height() - target_height;
DCHECK(stack_state.empty());
DCHECK_GE(source.stack_height(), stack_base);
stack_state.resize_no_init(target_height);
const VarState* source_begin = source.stack_state.data();
VarState* target_begin = stack_state.data();
// Try to keep locals and the merge region in their registers. Register used
// multiple times need to be copied to another free register. Compute the list
// of used registers.
LiftoffRegList used_regs;
for (auto& src : VectorOf(source_begin, num_locals)) {
if (src.is_reg()) used_regs.set(src.reg());
}
for (auto& src : VectorOf(source_begin + stack_base + discarded, arity)) {
if (src.is_reg()) used_regs.set(src.reg());
}
// Initialize the merge region. If this region moves, try to turn stack slots
// into registers since we need to load the value anyways.
MergeKeepStackSlots keep_merge_stack_slots =
discarded == 0 ? kKeepStackSlots : kTurnStackSlotsIntoRegisters;
InitMergeRegion(this, source_begin + stack_base + discarded,
target_begin + stack_base, arity, keep_merge_stack_slots,
kConstantsNotAllowed, kNoReuseRegisters, used_regs);
// Initialize the locals region. Here, stack slots stay stack slots (because
// they do not move). Try to keep register in registers, but avoid duplicates.
InitMergeRegion(this, source_begin, target_begin, num_locals, kKeepStackSlots,
kConstantsNotAllowed, kNoReuseRegisters, used_regs);
// Sanity check: All the {used_regs} are really in use now.
DCHECK_EQ(used_regs, used_registers & used_regs);
// Last, initialize the section in between. Here, constants are allowed, but
// registers which are already used for the merge region or locals must be
// moved to other registers or spilled. If a register appears twice in the
// source region, ensure to use the same register twice in the target region.
InitMergeRegion(this, source_begin + num_locals, target_begin + num_locals,
stack_depth, kKeepStackSlots, kConstantsAllowed,
kReuseRegisters, used_regs);
}
void LiftoffAssembler::CacheState::Steal(const CacheState& source) {
// Just use the move assignment operator.
*this = std::move(source);
}
void LiftoffAssembler::CacheState::Split(const CacheState& source) {
// Call the private copy assignment operator.
*this = source;
}
namespace {
constexpr AssemblerOptions DefaultLiftoffOptions() {
return AssemblerOptions{};
}
} // namespace
// TODO(clemensh): Provide a reasonably sized buffer, based on wasm function
// size.
LiftoffAssembler::LiftoffAssembler(std::unique_ptr<AssemblerBuffer> buffer)
: TurboAssembler(nullptr, DefaultLiftoffOptions(), CodeObjectRequired::kNo,
std::move(buffer)) {
set_abort_hard(true); // Avoid calls to Abort.
}
LiftoffAssembler::~LiftoffAssembler() {
if (num_locals_ > kInlineLocalTypes) {
free(more_local_types_);
}
}
LiftoffRegister LiftoffAssembler::PopToRegister(LiftoffRegList pinned) {
DCHECK(!cache_state_.stack_state.empty());
VarState slot = cache_state_.stack_state.back();
cache_state_.stack_state.pop_back();
switch (slot.loc()) {
case VarState::kStack: {
LiftoffRegister reg =
GetUnusedRegister(reg_class_for(slot.type()), pinned);
Fill(reg, cache_state_.stack_height(), slot.type());
return reg;
}
case VarState::kRegister:
cache_state_.dec_used(slot.reg());
return slot.reg();
case VarState::kIntConst: {
RegClass rc =
kNeedI64RegPair && slot.type() == kWasmI64 ? kGpRegPair : kGpReg;
LiftoffRegister reg = GetUnusedRegister(rc, pinned);
LoadConstant(reg, slot.constant());
return reg;
}
}
UNREACHABLE();
}
void LiftoffAssembler::MergeFullStackWith(const CacheState& target,
const CacheState& source) {
DCHECK_EQ(source.stack_height(), target.stack_height());
// TODO(clemensh): Reuse the same StackTransferRecipe object to save some
// allocations.
StackTransferRecipe transfers(this);
for (uint32_t i = 0, e = source.stack_height(); i < e; ++i) {
transfers.TransferStackSlot(target, i, source, i);
}
}
void LiftoffAssembler::MergeStackWith(const CacheState& target,
uint32_t arity) {
// Before: ----------------|----- (discarded) ----|--- arity ---|
// ^target_stack_height ^stack_base ^stack_height
// After: ----|-- arity --|
// ^ ^target_stack_height
// ^target_stack_base
uint32_t stack_height = cache_state_.stack_height();
uint32_t target_stack_height = target.stack_height();
DCHECK_LE(target_stack_height, stack_height);
DCHECK_LE(arity, target_stack_height);
uint32_t stack_base = stack_height - arity;
uint32_t target_stack_base = target_stack_height - arity;
StackTransferRecipe transfers(this);
for (uint32_t i = 0; i < target_stack_base; ++i) {
transfers.TransferStackSlot(target, i, cache_state_, i);
}
for (uint32_t i = 0; i < arity; ++i) {
transfers.TransferStackSlot(target, target_stack_base + i, cache_state_,
stack_base + i);
}
}
void LiftoffAssembler::Spill(uint32_t index) {
auto& slot = cache_state_.stack_state[index];
switch (slot.loc()) {
case VarState::kStack:
return;
case VarState::kRegister:
Spill(index, slot.reg(), slot.type());
cache_state_.dec_used(slot.reg());
break;
case VarState::kIntConst:
Spill(index, slot.constant());
break;
}
slot.MakeStack();
}
void LiftoffAssembler::SpillLocals() {
for (uint32_t i = 0; i < num_locals_; ++i) {
Spill(i);
}
}
void LiftoffAssembler::SpillAllRegisters() {
for (uint32_t i = 0, e = cache_state_.stack_height(); i < e; ++i) {
auto& slot = cache_state_.stack_state[i];
if (!slot.is_reg()) continue;
Spill(i, slot.reg(), slot.type());
slot.MakeStack();
}
cache_state_.reset_used_registers();
}
void LiftoffAssembler::PrepareCall(FunctionSig* sig,
compiler::CallDescriptor* call_descriptor,
Register* target,
Register* target_instance) {
uint32_t num_params = static_cast<uint32_t>(sig->parameter_count());
// Input 0 is the call target.
constexpr size_t kInputShift = 1;
// Spill all cache slots which are not being used as parameters.
// Don't update any register use counters, they will be reset later anyway.
for (uint32_t idx = 0, end = cache_state_.stack_height() - num_params;
idx < end; ++idx) {
VarState& slot = cache_state_.stack_state[idx];
if (!slot.is_reg()) continue;
Spill(idx, slot.reg(), slot.type());
slot.MakeStack();
}
LiftoffStackSlots stack_slots(this);
StackTransferRecipe stack_transfers(this);
LiftoffRegList param_regs;
// Move the target instance (if supplied) into the correct instance register.
compiler::LinkageLocation instance_loc =
call_descriptor->GetInputLocation(kInputShift);
DCHECK(instance_loc.IsRegister() && !instance_loc.IsAnyRegister());
Register instance_reg = Register::from_code(instance_loc.AsRegister());
param_regs.set(instance_reg);
if (target_instance && *target_instance != instance_reg) {
stack_transfers.MoveRegister(LiftoffRegister(instance_reg),
LiftoffRegister(*target_instance),
kWasmIntPtr);
}
// Now move all parameter values into the right slot for the call.
// Don't pop values yet, such that the stack height is still correct when
// executing the {stack_transfers}.
// Process parameters backwards, such that pushes of caller frame slots are
// in the correct order.
uint32_t param_base = cache_state_.stack_height() - num_params;
uint32_t call_desc_input_idx =
static_cast<uint32_t>(call_descriptor->InputCount());
for (uint32_t i = num_params; i > 0; --i) {
const uint32_t param = i - 1;
ValueType type = sig->GetParam(param);
const bool is_pair = kNeedI64RegPair && type == kWasmI64;
const int num_lowered_params = is_pair ? 2 : 1;
const uint32_t stack_idx = param_base + param;
const VarState& slot = cache_state_.stack_state[stack_idx];
// Process both halfs of a register pair separately, because they are passed
// as separate parameters. One or both of them could end up on the stack.
for (int lowered_idx = 0; lowered_idx < num_lowered_params; ++lowered_idx) {
const RegPairHalf half =
is_pair && lowered_idx == 0 ? kHighWord : kLowWord;
--call_desc_input_idx;
compiler::LinkageLocation loc =
call_descriptor->GetInputLocation(call_desc_input_idx);
if (loc.IsRegister()) {
DCHECK(!loc.IsAnyRegister());
RegClass rc = is_pair ? kGpReg : reg_class_for(type);
int reg_code = loc.AsRegister();
#if V8_TARGET_ARCH_ARM
// Liftoff assumes a one-to-one mapping between float registers and
// double registers, and so does not distinguish between f32 and f64
// registers. The f32 register code must therefore be halved in order to
// pass the f64 code to Liftoff.
DCHECK_IMPLIES(type == kWasmF32, (reg_code % 2) == 0);
LiftoffRegister reg = LiftoffRegister::from_code(
rc, (type == kWasmF32) ? (reg_code / 2) : reg_code);
#else
LiftoffRegister reg = LiftoffRegister::from_code(rc, reg_code);
#endif
param_regs.set(reg);
if (is_pair) {
stack_transfers.LoadI64HalfIntoRegister(reg, slot, stack_idx, half);
} else {
stack_transfers.LoadIntoRegister(reg, slot, stack_idx);
}
} else {
DCHECK(loc.IsCallerFrameSlot());
stack_slots.Add(slot, stack_idx, half);
}
}
}
// {call_desc_input_idx} should point after the instance parameter now.
DCHECK_EQ(call_desc_input_idx, kInputShift + 1);
// If the target register overlaps with a parameter register, then move the
// target to another free register, or spill to the stack.
if (target && param_regs.has(LiftoffRegister(*target))) {
// Try to find another free register.
LiftoffRegList free_regs = kGpCacheRegList.MaskOut(param_regs);
if (!free_regs.is_empty()) {
LiftoffRegister new_target = free_regs.GetFirstRegSet();
stack_transfers.MoveRegister(new_target, LiftoffRegister(*target),
kWasmIntPtr);
*target = new_target.gp();
} else {
stack_slots.Add(LiftoffAssembler::VarState(LiftoffAssembler::kWasmIntPtr,
LiftoffRegister(*target)));
*target = no_reg;
}
}
// Create all the slots.
stack_slots.Construct();
// Execute the stack transfers before filling the instance register.
stack_transfers.Execute();
// Pop parameters from the value stack.
cache_state_.stack_state.pop_back(num_params);
// Reset register use counters.
cache_state_.reset_used_registers();
// Reload the instance from the stack.
if (!target_instance) {
FillInstanceInto(instance_reg);
}
}
void LiftoffAssembler::FinishCall(FunctionSig* sig,
compiler::CallDescriptor* call_descriptor) {
const size_t return_count = sig->return_count();
if (return_count != 0) {
DCHECK_EQ(1, return_count);
ValueType return_type = sig->GetReturn(0);
const bool need_pair = kNeedI64RegPair && return_type == kWasmI64;
DCHECK_EQ(need_pair ? 2 : 1, call_descriptor->ReturnCount());
RegClass rc = need_pair ? kGpReg : reg_class_for(return_type);
#if V8_TARGET_ARCH_ARM
// If the return register was not d0 for f32, the code value would have to
// be halved as is done for the parameter registers.
DCHECK_EQ(call_descriptor->GetReturnLocation(0).AsRegister(), 0);
#endif
LiftoffRegister return_reg = LiftoffRegister::from_code(
rc, call_descriptor->GetReturnLocation(0).AsRegister());
DCHECK(GetCacheRegList(rc).has(return_reg));
if (need_pair) {
LiftoffRegister high_reg = LiftoffRegister::from_code(
rc, call_descriptor->GetReturnLocation(1).AsRegister());
DCHECK(GetCacheRegList(rc).has(high_reg));
return_reg = LiftoffRegister::ForPair(return_reg.gp(), high_reg.gp());
}
DCHECK(!cache_state_.is_used(return_reg));
PushRegister(return_type, return_reg);
}
}
void LiftoffAssembler::Move(LiftoffRegister dst, LiftoffRegister src,
ValueType type) {
DCHECK_EQ(dst.reg_class(), src.reg_class());
DCHECK_NE(dst, src);
if (kNeedI64RegPair && dst.is_pair()) {
// Use the {StackTransferRecipe} to move pairs, as the registers in the
// pairs might overlap.
StackTransferRecipe(this).MoveRegister(dst, src, type);
} else if (dst.is_gp()) {
Move(dst.gp(), src.gp(), type);
} else {
Move(dst.fp(), src.fp(), type);
}
}
void LiftoffAssembler::ParallelRegisterMove(
Vector<ParallelRegisterMoveTuple> tuples) {
StackTransferRecipe stack_transfers(this);
for (auto tuple : tuples) {
if (tuple.dst == tuple.src) continue;
stack_transfers.MoveRegister(tuple.dst, tuple.src, tuple.type);
}
}
void LiftoffAssembler::MoveToReturnRegisters(FunctionSig* sig) {
// We do not support multi-value yet.
DCHECK_EQ(1, sig->return_count());
ValueType return_type = sig->GetReturn(0);
StackTransferRecipe stack_transfers(this);
LiftoffRegister return_reg =
needs_reg_pair(return_type)
? LiftoffRegister::ForPair(kGpReturnRegisters[0],
kGpReturnRegisters[1])
: reg_class_for(return_type) == kGpReg
? LiftoffRegister(kGpReturnRegisters[0])
: LiftoffRegister(kFpReturnRegisters[0]);
stack_transfers.LoadIntoRegister(return_reg, cache_state_.stack_state.back(),
cache_state_.stack_height() - 1);
}
#ifdef ENABLE_SLOW_DCHECKS
bool LiftoffAssembler::ValidateCacheState() const {
uint32_t register_use_count[kAfterMaxLiftoffRegCode] = {0};
LiftoffRegList used_regs;
for (const VarState& var : cache_state_.stack_state) {
if (!var.is_reg()) continue;
LiftoffRegister reg = var.reg();
if (kNeedI64RegPair && reg.is_pair()) {
++register_use_count[reg.low().liftoff_code()];
++register_use_count[reg.high().liftoff_code()];
} else {
++register_use_count[reg.liftoff_code()];
}
used_regs.set(reg);
}
bool valid = memcmp(register_use_count, cache_state_.register_use_count,
sizeof(register_use_count)) == 0 &&
used_regs == cache_state_.used_registers;
if (valid) return true;
std::ostringstream os;
os << "Error in LiftoffAssembler::ValidateCacheState().\n";
os << "expected: used_regs " << used_regs << ", counts "
<< PrintCollection(register_use_count) << "\n";
os << "found: used_regs " << cache_state_.used_registers << ", counts "
<< PrintCollection(cache_state_.register_use_count) << "\n";
os << "Use --trace-wasm-decoder and --trace-liftoff to debug.";
FATAL("%s", os.str().c_str());
}
#endif
LiftoffRegister LiftoffAssembler::SpillOneRegister(LiftoffRegList candidates,
LiftoffRegList pinned) {
// Spill one cached value to free a register.
LiftoffRegister spill_reg = cache_state_.GetNextSpillReg(candidates, pinned);
SpillRegister(spill_reg);
return spill_reg;
}
void LiftoffAssembler::SpillRegister(LiftoffRegister reg) {
int remaining_uses = cache_state_.get_use_count(reg);
DCHECK_LT(0, remaining_uses);
for (uint32_t idx = cache_state_.stack_height() - 1;; --idx) {
DCHECK_GT(cache_state_.stack_height(), idx);
auto* slot = &cache_state_.stack_state[idx];
if (!slot->is_reg() || !slot->reg().overlaps(reg)) continue;
if (slot->reg().is_pair()) {
// Make sure to decrement *both* registers in a pair, because the
// {clear_used} call below only clears one of them.
cache_state_.dec_used(slot->reg().low());
cache_state_.dec_used(slot->reg().high());
}
Spill(idx, slot->reg(), slot->type());
slot->MakeStack();
if (--remaining_uses == 0) break;
}
cache_state_.clear_used(reg);
}
void LiftoffAssembler::set_num_locals(uint32_t num_locals) {
DCHECK_EQ(0, num_locals_); // only call this once.
num_locals_ = num_locals;
if (num_locals > kInlineLocalTypes) {
more_local_types_ =
reinterpret_cast<ValueType*>(malloc(num_locals * sizeof(ValueType)));
DCHECK_NOT_NULL(more_local_types_);
}
}
std::ostream& operator<<(std::ostream& os, VarState slot) {
os << ValueTypes::TypeName(slot.type()) << ":";
switch (slot.loc()) {
case VarState::kStack:
return os << "s";
case VarState::kRegister:
return os << slot.reg();
case VarState::kIntConst:
return os << "c" << slot.i32_const();
}
UNREACHABLE();
}
#undef __
#undef TRACE
} // namespace wasm
} // namespace internal
} // namespace v8