Server : Apache/2.4.6 (CentOS) OpenSSL/1.0.2k-fips PHP/7.4.20 System : Linux st2.domain.com 3.10.0-1127.10.1.el7.x86_64 #1 SMP Wed Jun 3 14:28:03 UTC 2020 x86_64 User : apache ( 48) PHP Version : 7.4.20 Disable Function : NONE Directory : /home/real/node-v13.0.1/deps/v8/src/heap/ |
// Copyright 2012 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "src/heap/incremental-marking.h"
#include "src/codegen/compilation-cache.h"
#include "src/execution/vm-state-inl.h"
#include "src/heap/concurrent-marking.h"
#include "src/heap/embedder-tracing.h"
#include "src/heap/gc-idle-time-handler.h"
#include "src/heap/gc-tracer.h"
#include "src/heap/heap-inl.h"
#include "src/heap/incremental-marking-inl.h"
#include "src/heap/mark-compact-inl.h"
#include "src/heap/object-stats.h"
#include "src/heap/objects-visiting-inl.h"
#include "src/heap/objects-visiting.h"
#include "src/heap/sweeper.h"
#include "src/init/v8.h"
#include "src/numbers/conversions.h"
#include "src/objects/data-handler-inl.h"
#include "src/objects/embedder-data-array-inl.h"
#include "src/objects/hash-table-inl.h"
#include "src/objects/slots-inl.h"
#include "src/objects/transitions-inl.h"
#include "src/objects/visitors.h"
#include "src/tracing/trace-event.h"
namespace v8 {
namespace internal {
using IncrementalMarkingMarkingVisitor =
MarkingVisitor<FixedArrayVisitationMode::kIncremental,
TraceRetainingPathMode::kDisabled,
IncrementalMarking::MarkingState>;
void IncrementalMarking::Observer::Step(int bytes_allocated, Address addr,
size_t size) {
Heap* heap = incremental_marking_->heap();
VMState<GC> state(heap->isolate());
RuntimeCallTimerScope runtime_timer(
heap->isolate(),
RuntimeCallCounterId::kGC_Custom_IncrementalMarkingObserver);
incremental_marking_->AdvanceOnAllocation();
// AdvanceIncrementalMarkingOnAllocation can start incremental marking.
incremental_marking_->EnsureBlackAllocated(addr, size);
}
IncrementalMarking::IncrementalMarking(
Heap* heap, MarkCompactCollector::MarkingWorklist* marking_worklist,
WeakObjects* weak_objects)
: heap_(heap),
marking_worklist_(marking_worklist),
weak_objects_(weak_objects),
initial_old_generation_size_(0),
bytes_marked_(0),
scheduled_bytes_to_mark_(0),
schedule_update_time_ms_(0),
bytes_marked_concurrently_(0),
is_compacting_(false),
should_hurry_(false),
was_activated_(false),
black_allocation_(false),
finalize_marking_completed_(false),
request_type_(NONE),
new_generation_observer_(this, kYoungGenerationAllocatedThreshold),
old_generation_observer_(this, kOldGenerationAllocatedThreshold) {
DCHECK_NOT_NULL(marking_worklist_);
SetState(STOPPED);
}
void IncrementalMarking::RecordWriteSlow(HeapObject obj, HeapObjectSlot slot,
HeapObject value) {
if (BaseRecordWrite(obj, value) && slot.address() != kNullAddress) {
// Object is not going to be rescanned we need to record the slot.
heap_->mark_compact_collector()->RecordSlot(obj, slot, value);
}
}
int IncrementalMarking::RecordWriteFromCode(Address raw_obj,
Address slot_address,
Isolate* isolate) {
HeapObject obj = HeapObject::cast(Object(raw_obj));
MaybeObjectSlot slot(slot_address);
isolate->heap()->incremental_marking()->RecordWrite(obj, slot, *slot);
// Called by RecordWriteCodeStubAssembler, which doesnt accept void type
return 0;
}
void IncrementalMarking::RecordWriteIntoCode(Code host, RelocInfo* rinfo,
HeapObject value) {
DCHECK(IsMarking());
if (BaseRecordWrite(host, value)) {
// Object is not going to be rescanned. We need to record the slot.
heap_->mark_compact_collector()->RecordRelocSlot(host, rinfo, value);
}
}
void IncrementalMarking::MarkBlackAndVisitObjectDueToLayoutChange(
HeapObject obj) {
TRACE_EVENT0("v8", "V8.GCIncrementalMarkingLayoutChange");
TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_INCREMENTAL_LAYOUT_CHANGE);
marking_state()->WhiteToGrey(obj);
if (marking_state()->GreyToBlack(obj)) {
RevisitObject(obj);
}
}
void IncrementalMarking::NotifyLeftTrimming(HeapObject from, HeapObject to) {
DCHECK(IsMarking());
DCHECK(MemoryChunk::FromHeapObject(from)->SweepingDone());
DCHECK_EQ(MemoryChunk::FromHeapObject(from), MemoryChunk::FromHeapObject(to));
DCHECK_NE(from, to);
MarkBit new_mark_bit = marking_state()->MarkBitFrom(to);
if (black_allocation() && Marking::IsBlack<kAtomicity>(new_mark_bit)) {
// Nothing to do if the object is in black area.
return;
}
MarkBlackAndVisitObjectDueToLayoutChange(from);
DCHECK(marking_state()->IsBlack(from));
// Mark the new address as black.
if (from.address() + kTaggedSize == to.address()) {
// The old and the new markbits overlap. The |to| object has the
// grey color. To make it black, we need to set the second bit.
DCHECK(new_mark_bit.Get<kAtomicity>());
new_mark_bit.Next().Set<kAtomicity>();
} else {
bool success = Marking::WhiteToBlack<kAtomicity>(new_mark_bit);
DCHECK(success);
USE(success);
}
DCHECK(marking_state()->IsBlack(to));
}
class IncrementalMarkingRootMarkingVisitor : public RootVisitor {
public:
explicit IncrementalMarkingRootMarkingVisitor(
IncrementalMarking* incremental_marking)
: heap_(incremental_marking->heap()) {}
void VisitRootPointer(Root root, const char* description,
FullObjectSlot p) override {
MarkObjectByPointer(p);
}
void VisitRootPointers(Root root, const char* description,
FullObjectSlot start, FullObjectSlot end) override {
for (FullObjectSlot p = start; p < end; ++p) MarkObjectByPointer(p);
}
private:
void MarkObjectByPointer(FullObjectSlot p) {
Object obj = *p;
if (!obj.IsHeapObject()) return;
heap_->incremental_marking()->WhiteToGreyAndPush(HeapObject::cast(obj));
}
Heap* heap_;
};
void IncrementalMarking::DeactivateIncrementalWriteBarrierForSpace(
PagedSpace* space) {
for (Page* p : *space) {
p->SetOldGenerationPageFlags(false);
}
}
void IncrementalMarking::DeactivateIncrementalWriteBarrierForSpace(
NewSpace* space) {
for (Page* p : *space) {
p->SetYoungGenerationPageFlags(false);
}
}
void IncrementalMarking::DeactivateIncrementalWriteBarrier() {
DeactivateIncrementalWriteBarrierForSpace(heap_->old_space());
DeactivateIncrementalWriteBarrierForSpace(heap_->map_space());
DeactivateIncrementalWriteBarrierForSpace(heap_->code_space());
DeactivateIncrementalWriteBarrierForSpace(heap_->new_space());
for (LargePage* p : *heap_->new_lo_space()) {
p->SetYoungGenerationPageFlags(false);
DCHECK(p->IsLargePage());
}
for (LargePage* p : *heap_->lo_space()) {
p->SetOldGenerationPageFlags(false);
}
for (LargePage* p : *heap_->code_lo_space()) {
p->SetOldGenerationPageFlags(false);
}
}
void IncrementalMarking::ActivateIncrementalWriteBarrier(PagedSpace* space) {
for (Page* p : *space) {
p->SetOldGenerationPageFlags(true);
}
}
void IncrementalMarking::ActivateIncrementalWriteBarrier(NewSpace* space) {
for (Page* p : *space) {
p->SetYoungGenerationPageFlags(true);
}
}
void IncrementalMarking::ActivateIncrementalWriteBarrier() {
ActivateIncrementalWriteBarrier(heap_->old_space());
ActivateIncrementalWriteBarrier(heap_->map_space());
ActivateIncrementalWriteBarrier(heap_->code_space());
ActivateIncrementalWriteBarrier(heap_->new_space());
for (LargePage* p : *heap_->new_lo_space()) {
p->SetYoungGenerationPageFlags(true);
DCHECK(p->IsLargePage());
}
for (LargePage* p : *heap_->lo_space()) {
p->SetOldGenerationPageFlags(true);
}
for (LargePage* p : *heap_->code_lo_space()) {
p->SetOldGenerationPageFlags(true);
}
}
bool IncrementalMarking::WasActivated() { return was_activated_; }
bool IncrementalMarking::CanBeActivated() {
// Only start incremental marking in a safe state: 1) when incremental
// marking is turned on, 2) when we are currently not in a GC, and
// 3) when we are currently not serializing or deserializing the heap.
return FLAG_incremental_marking && heap_->gc_state() == Heap::NOT_IN_GC &&
heap_->deserialization_complete() &&
!heap_->isolate()->serializer_enabled();
}
bool IncrementalMarking::IsBelowActivationThresholds() const {
return heap_->OldGenerationSizeOfObjects() <= kV8ActivationThreshold &&
heap_->GlobalSizeOfObjects() <= kGlobalActivationThreshold;
}
void IncrementalMarking::Deactivate() {
DeactivateIncrementalWriteBarrier();
}
void IncrementalMarking::Start(GarbageCollectionReason gc_reason) {
if (FLAG_trace_incremental_marking) {
const size_t old_generation_size_mb =
heap()->OldGenerationSizeOfObjects() / MB;
const size_t old_generation_limit_mb =
heap()->old_generation_allocation_limit() / MB;
const size_t global_size_mb = heap()->GlobalSizeOfObjects() / MB;
const size_t global_limit_mb = heap()->global_allocation_limit() / MB;
heap()->isolate()->PrintWithTimestamp(
"[IncrementalMarking] Start (%s): (size/limit/slack) v8: %zuMB / %zuMB "
"/ %zuMB global: %zuMB / %zuMB / %zuMB\n",
Heap::GarbageCollectionReasonToString(gc_reason),
old_generation_size_mb, old_generation_limit_mb,
old_generation_size_mb > old_generation_limit_mb
? 0
: old_generation_limit_mb - old_generation_size_mb,
global_size_mb, global_limit_mb,
global_size_mb > global_limit_mb ? 0
: global_limit_mb - global_size_mb);
}
DCHECK(FLAG_incremental_marking);
DCHECK(state_ == STOPPED);
DCHECK(heap_->gc_state() == Heap::NOT_IN_GC);
DCHECK(!heap_->isolate()->serializer_enabled());
Counters* counters = heap_->isolate()->counters();
counters->incremental_marking_reason()->AddSample(
static_cast<int>(gc_reason));
HistogramTimerScope incremental_marking_scope(
counters->gc_incremental_marking_start());
TRACE_EVENT0("v8", "V8.GCIncrementalMarkingStart");
TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_INCREMENTAL_START);
heap_->tracer()->NotifyIncrementalMarkingStart();
start_time_ms_ = heap()->MonotonicallyIncreasingTimeInMs();
initial_old_generation_size_ = heap_->OldGenerationSizeOfObjects();
old_generation_allocation_counter_ = heap_->OldGenerationAllocationCounter();
bytes_marked_ = 0;
scheduled_bytes_to_mark_ = 0;
schedule_update_time_ms_ = start_time_ms_;
bytes_marked_concurrently_ = 0;
should_hurry_ = false;
was_activated_ = true;
if (!heap_->mark_compact_collector()->sweeping_in_progress()) {
StartMarking();
} else {
if (FLAG_trace_incremental_marking) {
heap()->isolate()->PrintWithTimestamp(
"[IncrementalMarking] Start sweeping.\n");
}
SetState(SWEEPING);
}
heap_->AddAllocationObserversToAllSpaces(&old_generation_observer_,
&new_generation_observer_);
incremental_marking_job()->Start(heap_);
}
void IncrementalMarking::StartMarking() {
if (heap_->isolate()->serializer_enabled()) {
// Black allocation currently starts when we start incremental marking,
// but we cannot enable black allocation while deserializing. Hence, we
// have to delay the start of incremental marking in that case.
if (FLAG_trace_incremental_marking) {
heap()->isolate()->PrintWithTimestamp(
"[IncrementalMarking] Start delayed - serializer\n");
}
return;
}
if (FLAG_trace_incremental_marking) {
heap()->isolate()->PrintWithTimestamp(
"[IncrementalMarking] Start marking\n");
}
is_compacting_ =
!FLAG_never_compact && heap_->mark_compact_collector()->StartCompaction();
SetState(MARKING);
ActivateIncrementalWriteBarrier();
// Marking bits are cleared by the sweeper.
#ifdef VERIFY_HEAP
if (FLAG_verify_heap) {
heap_->mark_compact_collector()->VerifyMarkbitsAreClean();
}
#endif
heap_->isolate()->compilation_cache()->MarkCompactPrologue();
StartBlackAllocation();
// Mark strong roots grey.
IncrementalMarkingRootMarkingVisitor visitor(this);
heap_->IterateStrongRoots(&visitor, VISIT_ONLY_STRONG);
if (FLAG_concurrent_marking && !heap_->IsTearingDown()) {
heap_->concurrent_marking()->ScheduleTasks();
}
// Ready to start incremental marking.
if (FLAG_trace_incremental_marking) {
heap()->isolate()->PrintWithTimestamp("[IncrementalMarking] Running\n");
}
{
// TracePrologue may call back into V8 in corner cases, requiring that
// marking (including write barriers) is fully set up.
TRACE_GC(heap()->tracer(),
GCTracer::Scope::MC_INCREMENTAL_EMBEDDER_PROLOGUE);
heap_->local_embedder_heap_tracer()->TracePrologue(
heap_->flags_for_embedder_tracer());
}
}
void IncrementalMarking::StartBlackAllocation() {
DCHECK(!black_allocation_);
DCHECK(IsMarking());
black_allocation_ = true;
heap()->old_space()->MarkLinearAllocationAreaBlack();
heap()->map_space()->MarkLinearAllocationAreaBlack();
heap()->code_space()->MarkLinearAllocationAreaBlack();
if (FLAG_trace_incremental_marking) {
heap()->isolate()->PrintWithTimestamp(
"[IncrementalMarking] Black allocation started\n");
}
}
void IncrementalMarking::PauseBlackAllocation() {
DCHECK(IsMarking());
heap()->old_space()->UnmarkLinearAllocationArea();
heap()->map_space()->UnmarkLinearAllocationArea();
heap()->code_space()->UnmarkLinearAllocationArea();
if (FLAG_trace_incremental_marking) {
heap()->isolate()->PrintWithTimestamp(
"[IncrementalMarking] Black allocation paused\n");
}
black_allocation_ = false;
}
void IncrementalMarking::FinishBlackAllocation() {
if (black_allocation_) {
black_allocation_ = false;
if (FLAG_trace_incremental_marking) {
heap()->isolate()->PrintWithTimestamp(
"[IncrementalMarking] Black allocation finished\n");
}
}
}
void IncrementalMarking::EnsureBlackAllocated(Address allocated, size_t size) {
if (black_allocation() && allocated != kNullAddress) {
HeapObject object = HeapObject::FromAddress(allocated);
if (marking_state()->IsWhite(object) && !Heap::InYoungGeneration(object)) {
if (heap_->IsLargeObject(object)) {
marking_state()->WhiteToBlack(object);
} else {
Page::FromAddress(allocated)->CreateBlackArea(allocated,
allocated + size);
}
}
}
}
void IncrementalMarking::MarkRoots() {
DCHECK(!finalize_marking_completed_);
DCHECK(IsMarking());
IncrementalMarkingRootMarkingVisitor visitor(this);
heap_->IterateStrongRoots(&visitor, VISIT_ONLY_STRONG);
}
bool IncrementalMarking::ShouldRetainMap(Map map, int age) {
if (age == 0) {
// The map has aged. Do not retain this map.
return false;
}
Object constructor = map.GetConstructor();
if (!constructor.IsHeapObject() ||
marking_state()->IsWhite(HeapObject::cast(constructor))) {
// The constructor is dead, no new objects with this map can
// be created. Do not retain this map.
return false;
}
return true;
}
void IncrementalMarking::RetainMaps() {
// Do not retain dead maps if flag disables it or there is
// - memory pressure (reduce_memory_footprint_),
// - GC is requested by tests or dev-tools (abort_incremental_marking_).
bool map_retaining_is_disabled = heap()->ShouldReduceMemory() ||
FLAG_retain_maps_for_n_gc == 0;
WeakArrayList retained_maps = heap()->retained_maps();
int length = retained_maps.length();
// The number_of_disposed_maps separates maps in the retained_maps
// array that were created before and after context disposal.
// We do not age and retain disposed maps to avoid memory leaks.
int number_of_disposed_maps = heap()->number_of_disposed_maps_;
for (int i = 0; i < length; i += 2) {
MaybeObject value = retained_maps.Get(i);
HeapObject map_heap_object;
if (!value->GetHeapObjectIfWeak(&map_heap_object)) {
continue;
}
int age = retained_maps.Get(i + 1).ToSmi().value();
int new_age;
Map map = Map::cast(map_heap_object);
if (i >= number_of_disposed_maps && !map_retaining_is_disabled &&
marking_state()->IsWhite(map)) {
if (ShouldRetainMap(map, age)) {
WhiteToGreyAndPush(map);
}
Object prototype = map.prototype();
if (age > 0 && prototype.IsHeapObject() &&
marking_state()->IsWhite(HeapObject::cast(prototype))) {
// The prototype is not marked, age the map.
new_age = age - 1;
} else {
// The prototype and the constructor are marked, this map keeps only
// transition tree alive, not JSObjects. Do not age the map.
new_age = age;
}
} else {
new_age = FLAG_retain_maps_for_n_gc;
}
// Compact the array and update the age.
if (new_age != age) {
retained_maps.Set(i + 1, MaybeObject::FromSmi(Smi::FromInt(new_age)));
}
}
}
void IncrementalMarking::FinalizeIncrementally() {
TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_INCREMENTAL_FINALIZE_BODY);
DCHECK(!finalize_marking_completed_);
DCHECK(IsMarking());
double start = heap_->MonotonicallyIncreasingTimeInMs();
// After finishing incremental marking, we try to discover all unmarked
// objects to reduce the marking load in the final pause.
// 1) We scan and mark the roots again to find all changes to the root set.
// 2) Age and retain maps embedded in optimized code.
MarkRoots();
// Map retaining is needed for perfromance, not correctness,
// so we can do it only once at the beginning of the finalization.
RetainMaps();
finalize_marking_completed_ = true;
if (FLAG_trace_incremental_marking) {
double end = heap_->MonotonicallyIncreasingTimeInMs();
double delta = end - start;
heap()->isolate()->PrintWithTimestamp(
"[IncrementalMarking] Finalize incrementally spent %.1f ms.\n", delta);
}
}
void IncrementalMarking::UpdateMarkingWorklistAfterScavenge() {
if (!IsMarking()) return;
Map filler_map = ReadOnlyRoots(heap_).one_pointer_filler_map();
#ifdef ENABLE_MINOR_MC
MinorMarkCompactCollector::MarkingState* minor_marking_state =
heap()->minor_mark_compact_collector()->marking_state();
#else
void* minor_marking_state = nullptr;
#endif // ENABLE_MINOR_MC
marking_worklist()->Update([
#ifdef DEBUG
// this is referred inside DCHECK.
this,
#endif
filler_map, minor_marking_state](
HeapObject obj, HeapObject* out) -> bool {
DCHECK(obj.IsHeapObject());
// Only pointers to from space have to be updated.
if (Heap::InFromPage(obj)) {
MapWord map_word = obj.map_word();
if (!map_word.IsForwardingAddress()) {
// There may be objects on the marking deque that do not exist anymore,
// e.g. left trimmed objects or objects from the root set (frames).
// If these object are dead at scavenging time, their marking deque
// entries will not point to forwarding addresses. Hence, we can discard
// them.
return false;
}
HeapObject dest = map_word.ToForwardingAddress();
DCHECK_IMPLIES(marking_state()->IsWhite(obj), obj.IsFiller());
*out = dest;
return true;
} else if (Heap::InToPage(obj)) {
// The object may be on a large page or on a page that was moved in new
// space.
DCHECK(Heap::IsLargeObject(obj) ||
Page::FromHeapObject(obj)->IsFlagSet(Page::SWEEP_TO_ITERATE));
#ifdef ENABLE_MINOR_MC
if (minor_marking_state->IsWhite(obj)) {
return false;
}
#endif // ENABLE_MINOR_MC
// Either a large object or an object marked by the minor mark-compactor.
*out = obj;
return true;
} else {
// The object may be on a page that was moved from new to old space. Only
// applicable during minor MC garbage collections.
if (Page::FromHeapObject(obj)->IsFlagSet(Page::SWEEP_TO_ITERATE)) {
#ifdef ENABLE_MINOR_MC
if (minor_marking_state->IsWhite(obj)) {
return false;
}
#endif // ENABLE_MINOR_MC
*out = obj;
return true;
}
DCHECK_IMPLIES(marking_state()->IsWhite(obj), obj.IsFiller());
// Skip one word filler objects that appear on the
// stack when we perform in place array shift.
if (obj.map() != filler_map) {
*out = obj;
return true;
}
return false;
}
});
UpdateWeakReferencesAfterScavenge();
}
void IncrementalMarking::UpdateWeakReferencesAfterScavenge() {
weak_objects_->weak_references.Update(
[](std::pair<HeapObject, HeapObjectSlot> slot_in,
std::pair<HeapObject, HeapObjectSlot>* slot_out) -> bool {
HeapObject heap_obj = slot_in.first;
HeapObject forwarded = ForwardingAddress(heap_obj);
if (!forwarded.is_null()) {
ptrdiff_t distance_to_slot =
slot_in.second.address() - slot_in.first.ptr();
Address new_slot = forwarded.ptr() + distance_to_slot;
slot_out->first = forwarded;
slot_out->second = HeapObjectSlot(new_slot);
return true;
}
return false;
});
weak_objects_->weak_objects_in_code.Update(
[](std::pair<HeapObject, Code> slot_in,
std::pair<HeapObject, Code>* slot_out) -> bool {
HeapObject heap_obj = slot_in.first;
HeapObject forwarded = ForwardingAddress(heap_obj);
if (!forwarded.is_null()) {
slot_out->first = forwarded;
slot_out->second = slot_in.second;
return true;
}
return false;
});
weak_objects_->ephemeron_hash_tables.Update(
[](EphemeronHashTable slot_in, EphemeronHashTable* slot_out) -> bool {
EphemeronHashTable forwarded = ForwardingAddress(slot_in);
if (!forwarded.is_null()) {
*slot_out = forwarded;
return true;
}
return false;
});
auto ephemeron_updater = [](Ephemeron slot_in, Ephemeron* slot_out) -> bool {
HeapObject key = slot_in.key;
HeapObject value = slot_in.value;
HeapObject forwarded_key = ForwardingAddress(key);
HeapObject forwarded_value = ForwardingAddress(value);
if (!forwarded_key.is_null() && !forwarded_value.is_null()) {
*slot_out = Ephemeron{forwarded_key, forwarded_value};
return true;
}
return false;
};
weak_objects_->current_ephemerons.Update(ephemeron_updater);
weak_objects_->next_ephemerons.Update(ephemeron_updater);
weak_objects_->discovered_ephemerons.Update(ephemeron_updater);
weak_objects_->flushed_js_functions.Update(
[](JSFunction slot_in, JSFunction* slot_out) -> bool {
JSFunction forwarded = ForwardingAddress(slot_in);
if (!forwarded.is_null()) {
*slot_out = forwarded;
return true;
}
return false;
});
#ifdef DEBUG
weak_objects_->bytecode_flushing_candidates.Iterate(
[](SharedFunctionInfo candidate) {
DCHECK(!Heap::InYoungGeneration(candidate));
});
#endif
}
void IncrementalMarking::UpdateMarkedBytesAfterScavenge(
size_t dead_bytes_in_new_space) {
if (!IsMarking()) return;
bytes_marked_ -= Min(bytes_marked_, dead_bytes_in_new_space);
}
int IncrementalMarking::VisitObject(Map map, HeapObject obj) {
DCHECK(marking_state()->IsGrey(obj) || marking_state()->IsBlack(obj));
if (!marking_state()->GreyToBlack(obj)) {
// The object can already be black in these cases:
// 1. The object is a fixed array with the progress bar.
// 2. The object is a JSObject that was colored black before
// unsafe layout change.
// 3. The object is a string that was colored black before
// unsafe layout change.
// 4. The object is materizalized by the deoptimizer.
// 5. The object is a descriptor array marked black by
// the descriptor array marking barrier.
DCHECK(obj.IsHashTable() || obj.IsPropertyArray() || obj.IsFixedArray() ||
obj.IsContext() || obj.IsJSObject() || obj.IsString() ||
obj.IsDescriptorArray());
}
DCHECK(marking_state()->IsBlack(obj));
WhiteToGreyAndPush(map);
IncrementalMarkingMarkingVisitor visitor(heap()->mark_compact_collector(),
marking_state());
return visitor.Visit(map, obj);
}
void IncrementalMarking::ProcessBlackAllocatedObject(HeapObject obj) {
if (IsMarking() && marking_state()->IsBlack(obj)) {
RevisitObject(obj);
}
}
void IncrementalMarking::RevisitObject(HeapObject obj) {
DCHECK(IsMarking());
DCHECK(marking_state()->IsBlack(obj));
DCHECK_IMPLIES(MemoryChunk::FromHeapObject(obj)->IsFlagSet(
MemoryChunk::HAS_PROGRESS_BAR),
0u == MemoryChunk::FromHeapObject(obj)->ProgressBar());
Map map = obj.map();
WhiteToGreyAndPush(map);
IncrementalMarkingMarkingVisitor visitor(heap()->mark_compact_collector(),
marking_state());
visitor.Visit(map, obj);
}
void IncrementalMarking::VisitDescriptors(HeapObject host,
DescriptorArray descriptors,
int number_of_own_descriptors) {
IncrementalMarkingMarkingVisitor visitor(heap()->mark_compact_collector(),
marking_state());
// This is necessary because the Scavenger records slots only for the
// promoted black objects and the marking visitor of DescriptorArray skips
// the descriptors marked by the visitor.VisitDescriptors() below.
visitor.MarkDescriptorArrayBlack(host, descriptors);
visitor.VisitDescriptors(descriptors, number_of_own_descriptors);
}
intptr_t IncrementalMarking::ProcessMarkingWorklist(
intptr_t bytes_to_process, ForceCompletionAction completion) {
intptr_t bytes_processed = 0;
while (bytes_processed < bytes_to_process || completion == FORCE_COMPLETION) {
HeapObject obj = marking_worklist()->Pop();
if (obj.is_null()) break;
// Left trimming may result in grey or black filler objects on the marking
// worklist. Ignore these objects.
if (obj.IsFiller()) {
// Due to copying mark bits and the fact that grey and black have their
// first bit set, one word fillers are always black.
DCHECK_IMPLIES(
obj.map() == ReadOnlyRoots(heap()).one_pointer_filler_map(),
marking_state()->IsBlack(obj));
// Other fillers may be black or grey depending on the color of the object
// that was trimmed.
DCHECK_IMPLIES(
obj.map() != ReadOnlyRoots(heap()).one_pointer_filler_map(),
marking_state()->IsBlackOrGrey(obj));
continue;
}
bytes_processed += VisitObject(obj.map(), obj);
}
return bytes_processed;
}
StepResult IncrementalMarking::EmbedderStep(double duration_ms) {
if (!ShouldDoEmbedderStep()) return StepResult::kNoImmediateWork;
constexpr size_t kObjectsToProcessBeforeInterrupt = 500;
TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_INCREMENTAL_EMBEDDER_TRACING);
double deadline = heap_->MonotonicallyIncreasingTimeInMs() + duration_ms;
bool empty_worklist;
do {
{
LocalEmbedderHeapTracer::ProcessingScope scope(
heap_->local_embedder_heap_tracer());
HeapObject object;
size_t cnt = 0;
empty_worklist = true;
while (marking_worklist()->embedder()->Pop(0, &object)) {
scope.TracePossibleWrapper(JSObject::cast(object));
if (++cnt == kObjectsToProcessBeforeInterrupt) {
cnt = 0;
empty_worklist = false;
break;
}
}
}
heap_->local_embedder_heap_tracer()->Trace(deadline);
} while (!empty_worklist &&
(heap_->MonotonicallyIncreasingTimeInMs() < deadline));
heap_->local_embedder_heap_tracer()->SetEmbedderWorklistEmpty(empty_worklist);
return empty_worklist ? StepResult::kNoImmediateWork
: StepResult::kMoreWorkRemaining;
}
void IncrementalMarking::Hurry() {
// A scavenge may have pushed new objects on the marking deque (due to black
// allocation) even in COMPLETE state. This may happen if scavenges are
// forced e.g. in tests. It should not happen when COMPLETE was set when
// incremental marking finished and a regular GC was triggered after that
// because should_hurry_ will force a full GC.
if (!marking_worklist()->IsEmpty()) {
double start = 0.0;
if (FLAG_trace_incremental_marking) {
start = heap_->MonotonicallyIncreasingTimeInMs();
if (FLAG_trace_incremental_marking) {
heap()->isolate()->PrintWithTimestamp("[IncrementalMarking] Hurry\n");
}
}
// TODO(gc) hurry can mark objects it encounters black as mutator
// was stopped.
ProcessMarkingWorklist(0, FORCE_COMPLETION);
SetState(COMPLETE);
if (FLAG_trace_incremental_marking) {
double end = heap_->MonotonicallyIncreasingTimeInMs();
double delta = end - start;
if (FLAG_trace_incremental_marking) {
heap()->isolate()->PrintWithTimestamp(
"[IncrementalMarking] Complete (hurry), spent %d ms.\n",
static_cast<int>(delta));
}
}
}
}
void IncrementalMarking::Stop() {
if (IsStopped()) return;
if (FLAG_trace_incremental_marking) {
int old_generation_size_mb =
static_cast<int>(heap()->OldGenerationSizeOfObjects() / MB);
int old_generation_limit_mb =
static_cast<int>(heap()->old_generation_allocation_limit() / MB);
heap()->isolate()->PrintWithTimestamp(
"[IncrementalMarking] Stopping: old generation %dMB, limit %dMB, "
"overshoot %dMB\n",
old_generation_size_mb, old_generation_limit_mb,
Max(0, old_generation_size_mb - old_generation_limit_mb));
}
SpaceIterator it(heap_);
while (it.HasNext()) {
Space* space = it.Next();
if (space == heap_->new_space()) {
space->RemoveAllocationObserver(&new_generation_observer_);
} else {
space->RemoveAllocationObserver(&old_generation_observer_);
}
}
IncrementalMarking::set_should_hurry(false);
heap_->isolate()->stack_guard()->ClearGC();
SetState(STOPPED);
is_compacting_ = false;
FinishBlackAllocation();
}
void IncrementalMarking::Finalize() {
Hurry();
Stop();
}
void IncrementalMarking::FinalizeMarking(CompletionAction action) {
DCHECK(!finalize_marking_completed_);
if (FLAG_trace_incremental_marking) {
heap()->isolate()->PrintWithTimestamp(
"[IncrementalMarking] requesting finalization of incremental "
"marking.\n");
}
request_type_ = FINALIZATION;
if (action == GC_VIA_STACK_GUARD) {
heap_->isolate()->stack_guard()->RequestGC();
}
}
void IncrementalMarking::MarkingComplete(CompletionAction action) {
SetState(COMPLETE);
// We will set the stack guard to request a GC now. This will mean the rest
// of the GC gets performed as soon as possible (we can't do a GC here in a
// record-write context). If a few things get allocated between now and then
// that shouldn't make us do a scavenge and keep being incremental, so we set
// the should-hurry flag to indicate that there can't be much work left to do.
set_should_hurry(true);
if (FLAG_trace_incremental_marking) {
heap()->isolate()->PrintWithTimestamp(
"[IncrementalMarking] Complete (normal).\n");
}
request_type_ = COMPLETE_MARKING;
if (action == GC_VIA_STACK_GUARD) {
heap_->isolate()->stack_guard()->RequestGC();
}
}
void IncrementalMarking::Epilogue() {
was_activated_ = false;
finalize_marking_completed_ = false;
}
bool IncrementalMarking::ShouldDoEmbedderStep() {
return state_ == MARKING && FLAG_incremental_marking_wrappers &&
heap_->local_embedder_heap_tracer()->InUse();
}
void IncrementalMarking::FastForwardSchedule() {
if (scheduled_bytes_to_mark_ < bytes_marked_) {
scheduled_bytes_to_mark_ = bytes_marked_;
if (FLAG_trace_incremental_marking) {
heap_->isolate()->PrintWithTimestamp(
"[IncrementalMarking] Fast-forwarded schedule\n");
}
}
}
void IncrementalMarking::FastForwardScheduleIfCloseToFinalization() {
// Consider marking close to finalization if 75% of the initial old
// generation was marked.
if (bytes_marked_ > 3 * (initial_old_generation_size_ / 4)) {
FastForwardSchedule();
}
}
void IncrementalMarking::ScheduleBytesToMarkBasedOnTime(double time_ms) {
// Time interval that should be sufficient to complete incremental marking.
constexpr double kTargetMarkingWallTimeInMs = 500;
constexpr double kMinTimeBetweenScheduleInMs = 10;
if (schedule_update_time_ms_ + kMinTimeBetweenScheduleInMs > time_ms) return;
double delta_ms =
Min(time_ms - schedule_update_time_ms_, kTargetMarkingWallTimeInMs);
schedule_update_time_ms_ = time_ms;
size_t bytes_to_mark =
(delta_ms / kTargetMarkingWallTimeInMs) * initial_old_generation_size_;
AddScheduledBytesToMark(bytes_to_mark);
if (FLAG_trace_incremental_marking) {
heap_->isolate()->PrintWithTimestamp(
"[IncrementalMarking] Scheduled %zuKB to mark based on time delta "
"%.1fms\n",
bytes_to_mark / KB, delta_ms);
}
}
namespace {
StepResult CombineStepResults(StepResult a, StepResult b) {
if (a == StepResult::kMoreWorkRemaining ||
b == StepResult::kMoreWorkRemaining)
return StepResult::kMoreWorkRemaining;
if (a == StepResult::kWaitingForFinalization ||
b == StepResult::kWaitingForFinalization)
return StepResult::kWaitingForFinalization;
return StepResult::kNoImmediateWork;
}
} // anonymous namespace
StepResult IncrementalMarking::AdvanceWithDeadline(
double deadline_in_ms, CompletionAction completion_action,
StepOrigin step_origin) {
HistogramTimerScope incremental_marking_scope(
heap_->isolate()->counters()->gc_incremental_marking());
TRACE_EVENT0("v8", "V8.GCIncrementalMarking");
TRACE_GC(heap_->tracer(), GCTracer::Scope::MC_INCREMENTAL);
DCHECK(!IsStopped());
ScheduleBytesToMarkBasedOnTime(heap()->MonotonicallyIncreasingTimeInMs());
FastForwardScheduleIfCloseToFinalization();
double remaining_time_in_ms = 0.0;
StepResult result;
do {
StepResult v8_result =
V8Step(kStepSizeInMs / 2, completion_action, step_origin);
remaining_time_in_ms =
deadline_in_ms - heap()->MonotonicallyIncreasingTimeInMs();
StepResult embedder_result =
EmbedderStep(Min(kStepSizeInMs, remaining_time_in_ms));
result = CombineStepResults(v8_result, embedder_result);
remaining_time_in_ms =
deadline_in_ms - heap()->MonotonicallyIncreasingTimeInMs();
} while (remaining_time_in_ms >= kStepSizeInMs &&
result == StepResult::kMoreWorkRemaining);
return result;
}
void IncrementalMarking::FinalizeSweeping() {
DCHECK(state_ == SWEEPING);
if (heap_->mark_compact_collector()->sweeping_in_progress() &&
(!FLAG_concurrent_sweeping ||
!heap_->mark_compact_collector()->sweeper()->AreSweeperTasksRunning())) {
heap_->mark_compact_collector()->EnsureSweepingCompleted();
}
if (!heap_->mark_compact_collector()->sweeping_in_progress()) {
#ifdef DEBUG
heap_->VerifyCountersAfterSweeping();
#endif
StartMarking();
}
}
size_t IncrementalMarking::StepSizeToKeepUpWithAllocations() {
// Update bytes_allocated_ based on the allocation counter.
size_t current_counter = heap_->OldGenerationAllocationCounter();
size_t result = current_counter - old_generation_allocation_counter_;
old_generation_allocation_counter_ = current_counter;
return result;
}
size_t IncrementalMarking::StepSizeToMakeProgress() {
const size_t kTargetStepCount = 256;
const size_t kTargetStepCountAtOOM = 32;
const size_t kMaxStepSizeInByte = 256 * KB;
size_t oom_slack = heap()->new_space()->Capacity() + 64 * MB;
if (!heap()->CanExpandOldGeneration(oom_slack)) {
return heap()->OldGenerationSizeOfObjects() / kTargetStepCountAtOOM;
}
return Min(Max(initial_old_generation_size_ / kTargetStepCount,
IncrementalMarking::kMinStepSizeInBytes),
kMaxStepSizeInByte);
}
void IncrementalMarking::AddScheduledBytesToMark(size_t bytes_to_mark) {
if (scheduled_bytes_to_mark_ + bytes_to_mark < scheduled_bytes_to_mark_) {
// The overflow case.
scheduled_bytes_to_mark_ = std::numeric_limits<std::size_t>::max();
} else {
scheduled_bytes_to_mark_ += bytes_to_mark;
}
}
void IncrementalMarking::ScheduleBytesToMarkBasedOnAllocation() {
size_t progress_bytes = StepSizeToMakeProgress();
size_t allocation_bytes = StepSizeToKeepUpWithAllocations();
size_t bytes_to_mark = progress_bytes + allocation_bytes;
AddScheduledBytesToMark(bytes_to_mark);
if (FLAG_trace_incremental_marking) {
heap_->isolate()->PrintWithTimestamp(
"[IncrementalMarking] Scheduled %zuKB to mark based on allocation "
"(progress=%zuKB, allocation=%zuKB)\n",
bytes_to_mark / KB, progress_bytes / KB, allocation_bytes / KB);
}
}
void IncrementalMarking::FetchBytesMarkedConcurrently() {
if (FLAG_concurrent_marking) {
size_t current_bytes_marked_concurrently =
heap()->concurrent_marking()->TotalMarkedBytes();
// The concurrent_marking()->TotalMarkedBytes() is not monothonic for a
// short period of time when a concurrent marking task is finishing.
if (current_bytes_marked_concurrently > bytes_marked_concurrently_) {
bytes_marked_ +=
current_bytes_marked_concurrently - bytes_marked_concurrently_;
bytes_marked_concurrently_ = current_bytes_marked_concurrently;
}
if (FLAG_trace_incremental_marking) {
heap_->isolate()->PrintWithTimestamp(
"[IncrementalMarking] Marked %zuKB on background threads\n",
heap_->concurrent_marking()->TotalMarkedBytes() / KB);
}
}
}
size_t IncrementalMarking::ComputeStepSizeInBytes(StepOrigin step_origin) {
FetchBytesMarkedConcurrently();
if (FLAG_trace_incremental_marking) {
if (scheduled_bytes_to_mark_ > bytes_marked_) {
heap_->isolate()->PrintWithTimestamp(
"[IncrementalMarking] Marker is %zuKB behind schedule\n",
(scheduled_bytes_to_mark_ - bytes_marked_) / KB);
} else {
heap_->isolate()->PrintWithTimestamp(
"[IncrementalMarking] Marker is %zuKB ahead of schedule\n",
(bytes_marked_ - scheduled_bytes_to_mark_) / KB);
}
}
// Allow steps on allocation to get behind the schedule by small ammount.
// This gives higher priority to steps in tasks.
size_t kScheduleMarginInBytes = step_origin == StepOrigin::kV8 ? 1 * MB : 0;
if (bytes_marked_ + kScheduleMarginInBytes > scheduled_bytes_to_mark_)
return 0;
return scheduled_bytes_to_mark_ - bytes_marked_ - kScheduleMarginInBytes;
}
void IncrementalMarking::AdvanceOnAllocation() {
// Code using an AlwaysAllocateScope assumes that the GC state does not
// change; that implies that no marking steps must be performed.
if (heap_->gc_state() != Heap::NOT_IN_GC || !FLAG_incremental_marking ||
(state_ != SWEEPING && state_ != MARKING) || heap_->always_allocate()) {
return;
}
HistogramTimerScope incremental_marking_scope(
heap_->isolate()->counters()->gc_incremental_marking());
TRACE_EVENT0("v8", "V8.GCIncrementalMarking");
TRACE_GC(heap_->tracer(), GCTracer::Scope::MC_INCREMENTAL);
ScheduleBytesToMarkBasedOnAllocation();
V8Step(kMaxStepSizeInMs, GC_VIA_STACK_GUARD, StepOrigin::kV8);
}
StepResult IncrementalMarking::V8Step(double max_step_size_in_ms,
CompletionAction action,
StepOrigin step_origin) {
StepResult result = StepResult::kMoreWorkRemaining;
double start = heap_->MonotonicallyIncreasingTimeInMs();
if (state_ == SWEEPING) {
TRACE_GC(heap_->tracer(), GCTracer::Scope::MC_INCREMENTAL_SWEEPING);
FinalizeSweeping();
}
size_t bytes_processed = 0, bytes_to_process = 0;
if (state_ == MARKING) {
if (FLAG_concurrent_marking) {
heap_->new_space()->ResetOriginalTop();
heap_->new_lo_space()->ResetPendingObject();
// It is safe to merge back all objects that were on hold to the shared
// work list at Step because we are at a safepoint where all objects
// are properly initialized.
marking_worklist()->shared()->MergeGlobalPool(
marking_worklist()->on_hold());
}
// Only print marking worklist in debug mode to save ~40KB of code size.
#ifdef DEBUG
if (FLAG_trace_incremental_marking && FLAG_trace_concurrent_marking &&
FLAG_trace_gc_verbose) {
marking_worklist()->Print();
}
#endif
if (FLAG_trace_incremental_marking) {
heap_->isolate()->PrintWithTimestamp(
"[IncrementalMarking] Marking speed %.fKB/ms\n",
heap()->tracer()->IncrementalMarkingSpeedInBytesPerMillisecond());
}
// The first step after Scavenge will see many allocated bytes.
// Cap the step size to distribute the marking work more uniformly.
size_t max_step_size = GCIdleTimeHandler::EstimateMarkingStepSize(
max_step_size_in_ms,
heap()->tracer()->IncrementalMarkingSpeedInBytesPerMillisecond());
bytes_to_process = Min(ComputeStepSizeInBytes(step_origin), max_step_size);
if (bytes_to_process == 0) {
result = StepResult::kNoImmediateWork;
}
bytes_processed =
ProcessMarkingWorklist(Max(bytes_to_process, kMinStepSizeInBytes));
bytes_marked_ += bytes_processed;
if (marking_worklist()->IsEmpty()) {
result = StepResult::kNoImmediateWork;
if (heap_->local_embedder_heap_tracer()
->ShouldFinalizeIncrementalMarking()) {
if (!finalize_marking_completed_) {
FinalizeMarking(action);
FastForwardSchedule();
result = StepResult::kWaitingForFinalization;
incremental_marking_job()->Start(heap_);
} else {
MarkingComplete(action);
result = StepResult::kWaitingForFinalization;
}
} else {
heap_->local_embedder_heap_tracer()->NotifyV8MarkingWorklistWasEmpty();
}
}
}
if (FLAG_concurrent_marking) {
marking_worklist()->ShareWorkIfGlobalPoolIsEmpty();
heap_->concurrent_marking()->RescheduleTasksIfNeeded();
}
double end = heap_->MonotonicallyIncreasingTimeInMs();
double duration = (end - start);
// Note that we report zero bytes here when sweeping was in progress or
// when we just started incremental marking. In these cases we did not
// process the marking deque.
heap_->tracer()->AddIncrementalMarkingStep(duration, bytes_processed);
if (FLAG_trace_incremental_marking) {
heap_->isolate()->PrintWithTimestamp(
"[IncrementalMarking] Step %s %zuKB (%zuKB) in %.1f\n",
step_origin == StepOrigin::kV8 ? "in v8" : "in task",
bytes_processed / KB, bytes_to_process / KB, duration);
}
return result;
}
} // namespace internal
} // namespace v8