���� JFIF �� � ( %"1"%)+...383,7(-.-
![]() Server : Apache/2.4.6 (CentOS) OpenSSL/1.0.2k-fips PHP/7.4.20 System : Linux st2.domain.com 3.10.0-1127.10.1.el7.x86_64 #1 SMP Wed Jun 3 14:28:03 UTC 2020 x86_64 User : apache ( 48) PHP Version : 7.4.20 Disable Function : NONE Directory : /home/real/node-v13.0.1/deps/v8/src/heap/ |
// Copyright 2017 the V8 project authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #include "src/heap/concurrent-marking.h" #include <stack> #include <unordered_map> #include "include/v8config.h" #include "src/base/template-utils.h" #include "src/execution/isolate.h" #include "src/heap/gc-tracer.h" #include "src/heap/heap-inl.h" #include "src/heap/heap.h" #include "src/heap/mark-compact-inl.h" #include "src/heap/mark-compact.h" #include "src/heap/marking.h" #include "src/heap/objects-visiting-inl.h" #include "src/heap/objects-visiting.h" #include "src/heap/worklist.h" #include "src/init/v8.h" #include "src/objects/data-handler-inl.h" #include "src/objects/embedder-data-array-inl.h" #include "src/objects/hash-table-inl.h" #include "src/objects/slots-inl.h" #include "src/objects/transitions-inl.h" #include "src/utils/utils-inl.h" #include "src/utils/utils.h" namespace v8 { namespace internal { class ConcurrentMarkingState final : public MarkingStateBase<ConcurrentMarkingState, AccessMode::ATOMIC> { public: explicit ConcurrentMarkingState(MemoryChunkDataMap* memory_chunk_data) : memory_chunk_data_(memory_chunk_data) {} ConcurrentBitmap<AccessMode::ATOMIC>* bitmap(const MemoryChunk* chunk) { DCHECK_EQ(reinterpret_cast<intptr_t>(&chunk->marking_bitmap_) - reinterpret_cast<intptr_t>(chunk), MemoryChunk::kMarkBitmapOffset); return chunk->marking_bitmap<AccessMode::ATOMIC>(); } void IncrementLiveBytes(MemoryChunk* chunk, intptr_t by) { (*memory_chunk_data_)[chunk].live_bytes += by; } // The live_bytes and SetLiveBytes methods of the marking state are // not used by the concurrent marker. private: MemoryChunkDataMap* memory_chunk_data_; }; // Helper class for storing in-object slot addresses and values. class SlotSnapshot { public: SlotSnapshot() : number_of_slots_(0) {} int number_of_slots() const { return number_of_slots_; } ObjectSlot slot(int i) const { return snapshot_[i].first; } Object value(int i) const { return snapshot_[i].second; } void clear() { number_of_slots_ = 0; } void add(ObjectSlot slot, Object value) { snapshot_[number_of_slots_++] = {slot, value}; } private: static const int kMaxSnapshotSize = JSObject::kMaxInstanceSize / kTaggedSize; int number_of_slots_; std::pair<ObjectSlot, Object> snapshot_[kMaxSnapshotSize]; DISALLOW_COPY_AND_ASSIGN(SlotSnapshot); }; class ConcurrentMarkingVisitor final : public HeapVisitor<int, ConcurrentMarkingVisitor> { public: using BaseClass = HeapVisitor<int, ConcurrentMarkingVisitor>; explicit ConcurrentMarkingVisitor( ConcurrentMarking::MarkingWorklist* shared, MemoryChunkDataMap* memory_chunk_data, WeakObjects* weak_objects, ConcurrentMarking::EmbedderTracingWorklist* embedder_objects, int task_id, bool embedder_tracing_enabled, unsigned mark_compact_epoch, bool is_forced_gc) : shared_(shared, task_id), weak_objects_(weak_objects), embedder_objects_(embedder_objects, task_id), marking_state_(memory_chunk_data), memory_chunk_data_(memory_chunk_data), task_id_(task_id), embedder_tracing_enabled_(embedder_tracing_enabled), mark_compact_epoch_(mark_compact_epoch), is_forced_gc_(is_forced_gc) { // It is not safe to access flags from concurrent marking visitor. So // set the bytecode flush mode based on the flags here bytecode_flush_mode_ = Heap::GetBytecodeFlushMode(); } template <typename T> static V8_INLINE T Cast(HeapObject object) { return T::cast(object); } bool ShouldVisit(HeapObject object) { return marking_state_.GreyToBlack(object); } bool AllowDefaultJSObjectVisit() { return false; } template <typename THeapObjectSlot> void ProcessStrongHeapObject(HeapObject host, THeapObjectSlot slot, HeapObject heap_object) { MarkObject(heap_object); MarkCompactCollector::RecordSlot(host, slot, heap_object); } template <typename THeapObjectSlot> void ProcessWeakHeapObject(HeapObject host, THeapObjectSlot slot, HeapObject heap_object) { #ifdef THREAD_SANITIZER MemoryChunk::FromHeapObject(heap_object)->SynchronizedHeapLoad(); #endif if (marking_state_.IsBlackOrGrey(heap_object)) { // Weak references with live values are directly processed here to // reduce the processing time of weak cells during the main GC // pause. MarkCompactCollector::RecordSlot(host, slot, heap_object); } else { // If we do not know about liveness of the value, we have to process // the reference when we know the liveness of the whole transitive // closure. weak_objects_->weak_references.Push(task_id_, std::make_pair(host, slot)); } } void VisitPointers(HeapObject host, ObjectSlot start, ObjectSlot end) override { VisitPointersImpl(host, start, end); } void VisitPointers(HeapObject host, MaybeObjectSlot start, MaybeObjectSlot end) override { VisitPointersImpl(host, start, end); } template <typename TSlot> V8_INLINE void VisitPointersImpl(HeapObject host, TSlot start, TSlot end) { using THeapObjectSlot = typename TSlot::THeapObjectSlot; for (TSlot slot = start; slot < end; ++slot) { typename TSlot::TObject object = slot.Relaxed_Load(); HeapObject heap_object; if (object.GetHeapObjectIfStrong(&heap_object)) { // If the reference changes concurrently from strong to weak, the write // barrier will treat the weak reference as strong, so we won't miss the // weak reference. ProcessStrongHeapObject(host, THeapObjectSlot(slot), heap_object); } else if (TSlot::kCanBeWeak && object.GetHeapObjectIfWeak(&heap_object)) { ProcessWeakHeapObject(host, THeapObjectSlot(slot), heap_object); } } } // Weak list pointers should be ignored during marking. The lists are // reconstructed after GC. void VisitCustomWeakPointers(HeapObject host, ObjectSlot start, ObjectSlot end) final {} void VisitEmbeddedPointer(Code host, RelocInfo* rinfo) final { DCHECK(RelocInfo::IsEmbeddedObjectMode(rinfo->rmode())); HeapObject object = rinfo->target_object(); RecordRelocSlot(host, rinfo, object); if (!marking_state_.IsBlackOrGrey(object)) { if (host.IsWeakObject(object)) { weak_objects_->weak_objects_in_code.Push(task_id_, std::make_pair(object, host)); } else { MarkObject(object); } } } void VisitCodeTarget(Code host, RelocInfo* rinfo) final { DCHECK(RelocInfo::IsCodeTargetMode(rinfo->rmode())); Code target = Code::GetCodeFromTargetAddress(rinfo->target_address()); RecordRelocSlot(host, rinfo, target); MarkObject(target); } void VisitPointersInSnapshot(HeapObject host, const SlotSnapshot& snapshot) { for (int i = 0; i < snapshot.number_of_slots(); i++) { ObjectSlot slot = snapshot.slot(i); Object object = snapshot.value(i); DCHECK(!HasWeakHeapObjectTag(object)); if (!object.IsHeapObject()) continue; HeapObject heap_object = HeapObject::cast(object); MarkObject(heap_object); MarkCompactCollector::RecordSlot(host, slot, heap_object); } } // =========================================================================== // JS object ================================================================= // =========================================================================== int VisitJSObject(Map map, JSObject object) { return VisitJSObjectSubclass(map, object); } int VisitJSObjectFast(Map map, JSObject object) { return VisitJSObjectSubclassFast(map, object); } int VisitWasmInstanceObject(Map map, WasmInstanceObject object) { return VisitJSObjectSubclass(map, object); } int VisitJSWeakRef(Map map, JSWeakRef weak_ref) { int size = VisitJSObjectSubclass(map, weak_ref); if (size == 0) { return 0; } if (weak_ref.target().IsHeapObject()) { HeapObject target = HeapObject::cast(weak_ref.target()); if (marking_state_.IsBlackOrGrey(target)) { // Record the slot inside the JSWeakRef, since the // VisitJSObjectSubclass above didn't visit it. ObjectSlot slot = weak_ref.RawField(JSWeakRef::kTargetOffset); MarkCompactCollector::RecordSlot(weak_ref, slot, target); } else { // JSWeakRef points to a potentially dead object. We have to process // them when we know the liveness of the whole transitive closure. weak_objects_->js_weak_refs.Push(task_id_, weak_ref); } } return size; } int VisitWeakCell(Map map, WeakCell weak_cell) { if (!ShouldVisit(weak_cell)) return 0; int size = WeakCell::BodyDescriptor::SizeOf(map, weak_cell); VisitMapPointer(weak_cell); WeakCell::BodyDescriptor::IterateBody(map, weak_cell, size, this); if (weak_cell.target().IsHeapObject()) { HeapObject target = HeapObject::cast(weak_cell.target()); if (marking_state_.IsBlackOrGrey(target)) { // Record the slot inside the WeakCell, since the IterateBody above // didn't visit it. ObjectSlot slot = weak_cell.RawField(WeakCell::kTargetOffset); MarkCompactCollector::RecordSlot(weak_cell, slot, target); } else { // WeakCell points to a potentially dead object. We have to process // them when we know the liveness of the whole transitive closure. weak_objects_->weak_cells.Push(task_id_, weak_cell); } } return size; } // Some JS objects can carry back links to embedders that contain information // relevant to the garbage collectors. int VisitJSApiObject(Map map, JSObject object) { return VisitEmbedderTracingSubclass(map, object); } int VisitJSArrayBuffer(Map map, JSArrayBuffer object) { return VisitEmbedderTracingSubclass(map, object); } int VisitJSDataView(Map map, JSDataView object) { return VisitEmbedderTracingSubclass(map, object); } int VisitJSTypedArray(Map map, JSTypedArray object) { return VisitEmbedderTracingSubclass(map, object); } // =========================================================================== // Strings with pointers ===================================================== // =========================================================================== int VisitConsString(Map map, ConsString object) { return VisitFullyWithSnapshot(map, object); } int VisitSlicedString(Map map, SlicedString object) { return VisitFullyWithSnapshot(map, object); } int VisitThinString(Map map, ThinString object) { return VisitFullyWithSnapshot(map, object); } // =========================================================================== // Strings without pointers ================================================== // =========================================================================== int VisitSeqOneByteString(Map map, SeqOneByteString object) { if (!ShouldVisit(object)) return 0; VisitMapPointer(object); return SeqOneByteString::SizeFor(object.synchronized_length()); } int VisitSeqTwoByteString(Map map, SeqTwoByteString object) { if (!ShouldVisit(object)) return 0; VisitMapPointer(object); return SeqTwoByteString::SizeFor(object.synchronized_length()); } // =========================================================================== // Fixed array object ======================================================== // =========================================================================== int VisitFixedArrayWithProgressBar(Map map, FixedArray object, MemoryChunk* chunk) { // The concurrent marker can process larger chunks than the main thread // marker. const int kProgressBarScanningChunk = RoundUp(kMaxRegularHeapObjectSize, kTaggedSize); DCHECK(marking_state_.IsBlackOrGrey(object)); marking_state_.GreyToBlack(object); int size = FixedArray::BodyDescriptor::SizeOf(map, object); size_t current_progress_bar = chunk->ProgressBar(); int start = static_cast<int>(current_progress_bar); if (start == 0) start = FixedArray::BodyDescriptor::kStartOffset; int end = Min(size, start + kProgressBarScanningChunk); if (start < end) { VisitPointers(object, object.RawField(start), object.RawField(end)); bool success = chunk->TrySetProgressBar(current_progress_bar, end); CHECK(success); if (end < size) { // The object can be pushed back onto the marking worklist only after // progress bar was updated. shared_.Push(object); } } return end - start; } int VisitFixedArray(Map map, FixedArray object) { // Arrays with the progress bar are not left-trimmable because they reside // in the large object space. MemoryChunk* chunk = MemoryChunk::FromHeapObject(object); return chunk->IsFlagSet<AccessMode::ATOMIC>(MemoryChunk::HAS_PROGRESS_BAR) ? VisitFixedArrayWithProgressBar(map, object, chunk) : VisitLeftTrimmableArray(map, object); } int VisitFixedDoubleArray(Map map, FixedDoubleArray object) { return VisitLeftTrimmableArray(map, object); } // =========================================================================== // Side-effectful visitation. // =========================================================================== int VisitSharedFunctionInfo(Map map, SharedFunctionInfo shared_info) { if (!ShouldVisit(shared_info)) return 0; int size = SharedFunctionInfo::BodyDescriptor::SizeOf(map, shared_info); VisitMapPointer(shared_info); SharedFunctionInfo::BodyDescriptor::IterateBody(map, shared_info, size, this); // If the SharedFunctionInfo has old bytecode, mark it as flushable, // otherwise visit the function data field strongly. if (shared_info.ShouldFlushBytecode(bytecode_flush_mode_)) { weak_objects_->bytecode_flushing_candidates.Push(task_id_, shared_info); } else { VisitPointer(shared_info, shared_info.RawField( SharedFunctionInfo::kFunctionDataOffset)); } return size; } int VisitBytecodeArray(Map map, BytecodeArray object) { if (!ShouldVisit(object)) return 0; int size = BytecodeArray::BodyDescriptor::SizeOf(map, object); VisitMapPointer(object); BytecodeArray::BodyDescriptor::IterateBody(map, object, size, this); if (!is_forced_gc_) { object.MakeOlder(); } return size; } int VisitJSFunction(Map map, JSFunction object) { int size = VisitJSObjectSubclass(map, object); // Check if the JSFunction needs reset due to bytecode being flushed. if (bytecode_flush_mode_ != BytecodeFlushMode::kDoNotFlushBytecode && object.NeedsResetDueToFlushedBytecode()) { weak_objects_->flushed_js_functions.Push(task_id_, object); } return size; } int VisitMap(Map meta_map, Map map) { if (!ShouldVisit(map)) return 0; int size = Map::BodyDescriptor::SizeOf(meta_map, map); if (map.CanTransition()) { // Maps that can transition share their descriptor arrays and require // special visiting logic to avoid memory leaks. // Since descriptor arrays are potentially shared, ensure that only the // descriptors that belong to this map are marked. The first time a // non-empty descriptor array is marked, its header is also visited. The // slot holding the descriptor array will be implicitly recorded when the // pointer fields of this map are visited. DescriptorArray descriptors = map.synchronized_instance_descriptors(); MarkDescriptorArrayBlack(descriptors); int number_of_own_descriptors = map.NumberOfOwnDescriptors(); if (number_of_own_descriptors) { // It is possible that the concurrent marker observes the // number_of_own_descriptors out of sync with the descriptors. In that // case the marking write barrier for the descriptor array will ensure // that all required descriptors are marked. The concurrent marker // just should avoid crashing in that case. That's why we need the // std::min<int>() below. VisitDescriptors(descriptors, std::min<int>(number_of_own_descriptors, descriptors.number_of_descriptors())); } // Mark the pointer fields of the Map. Since the transitions array has // been marked already, it is fine that one of these fields contains a // pointer to it. } Map::BodyDescriptor::IterateBody(meta_map, map, size, this); return size; } void VisitDescriptors(DescriptorArray descriptor_array, int number_of_own_descriptors) { int16_t new_marked = static_cast<int16_t>(number_of_own_descriptors); int16_t old_marked = descriptor_array.UpdateNumberOfMarkedDescriptors( mark_compact_epoch_, new_marked); if (old_marked < new_marked) { VisitPointers( descriptor_array, MaybeObjectSlot(descriptor_array.GetDescriptorSlot(old_marked)), MaybeObjectSlot(descriptor_array.GetDescriptorSlot(new_marked))); } } int VisitDescriptorArray(Map map, DescriptorArray array) { if (!ShouldVisit(array)) return 0; VisitMapPointer(array); int size = DescriptorArray::BodyDescriptor::SizeOf(map, array); VisitPointers(array, array.GetFirstPointerSlot(), array.GetDescriptorSlot(0)); VisitDescriptors(array, array.number_of_descriptors()); return size; } int VisitTransitionArray(Map map, TransitionArray array) { if (!ShouldVisit(array)) return 0; VisitMapPointer(array); int size = TransitionArray::BodyDescriptor::SizeOf(map, array); TransitionArray::BodyDescriptor::IterateBody(map, array, size, this); weak_objects_->transition_arrays.Push(task_id_, array); return size; } int VisitJSWeakCollection(Map map, JSWeakCollection object) { return VisitJSObjectSubclass(map, object); } int VisitEphemeronHashTable(Map map, EphemeronHashTable table) { if (!ShouldVisit(table)) return 0; weak_objects_->ephemeron_hash_tables.Push(task_id_, table); for (int i = 0; i < table.Capacity(); i++) { ObjectSlot key_slot = table.RawFieldOfElementAt(EphemeronHashTable::EntryToIndex(i)); HeapObject key = HeapObject::cast(table.KeyAt(i)); MarkCompactCollector::RecordSlot(table, key_slot, key); ObjectSlot value_slot = table.RawFieldOfElementAt(EphemeronHashTable::EntryToValueIndex(i)); if (marking_state_.IsBlackOrGrey(key)) { VisitPointer(table, value_slot); } else { Object value_obj = table.ValueAt(i); if (value_obj.IsHeapObject()) { HeapObject value = HeapObject::cast(value_obj); MarkCompactCollector::RecordSlot(table, value_slot, value); // Revisit ephemerons with both key and value unreachable at end // of concurrent marking cycle. if (marking_state_.IsWhite(value)) { weak_objects_->discovered_ephemerons.Push(task_id_, Ephemeron{key, value}); } } } } return table.SizeFromMap(map); } // Implements ephemeron semantics: Marks value if key is already reachable. // Returns true if value was actually marked. bool ProcessEphemeron(HeapObject key, HeapObject value) { if (marking_state_.IsBlackOrGrey(key)) { if (marking_state_.WhiteToGrey(value)) { shared_.Push(value); return true; } } else if (marking_state_.IsWhite(value)) { weak_objects_->next_ephemerons.Push(task_id_, Ephemeron{key, value}); } return false; } void MarkObject(HeapObject object) { #ifdef THREAD_SANITIZER MemoryChunk::FromHeapObject(object)->SynchronizedHeapLoad(); #endif if (marking_state_.WhiteToGrey(object)) { shared_.Push(object); } } void MarkDescriptorArrayBlack(DescriptorArray descriptors) { marking_state_.WhiteToGrey(descriptors); if (marking_state_.GreyToBlack(descriptors)) { VisitPointers(descriptors, descriptors.GetFirstPointerSlot(), descriptors.GetDescriptorSlot(0)); } } private: // Helper class for collecting in-object slot addresses and values. class SlotSnapshottingVisitor final : public ObjectVisitor { public: explicit SlotSnapshottingVisitor(SlotSnapshot* slot_snapshot) : slot_snapshot_(slot_snapshot) { slot_snapshot_->clear(); } void VisitPointers(HeapObject host, ObjectSlot start, ObjectSlot end) override { for (ObjectSlot p = start; p < end; ++p) { Object object = p.Relaxed_Load(); slot_snapshot_->add(p, object); } } void VisitPointers(HeapObject host, MaybeObjectSlot start, MaybeObjectSlot end) override { // This should never happen, because we don't use snapshotting for objects // which contain weak references. UNREACHABLE(); } void VisitCodeTarget(Code host, RelocInfo* rinfo) final { // This should never happen, because snapshotting is performed only on // JSObjects (and derived classes). UNREACHABLE(); } void VisitEmbeddedPointer(Code host, RelocInfo* rinfo) final { // This should never happen, because snapshotting is performed only on // JSObjects (and derived classes). UNREACHABLE(); } void VisitCustomWeakPointers(HeapObject host, ObjectSlot start, ObjectSlot end) override { DCHECK(host.IsWeakCell() || host.IsJSWeakRef()); } private: SlotSnapshot* slot_snapshot_; }; template <typename T> int VisitJSObjectSubclassFast(Map map, T object) { DCHECK_IMPLIES(FLAG_unbox_double_fields, map.HasFastPointerLayout()); using TBodyDescriptor = typename T::FastBodyDescriptor; return VisitJSObjectSubclass<T, TBodyDescriptor>(map, object); } template <typename T, typename TBodyDescriptor = typename T::BodyDescriptor> int VisitJSObjectSubclass(Map map, T object) { int size = TBodyDescriptor::SizeOf(map, object); int used_size = map.UsedInstanceSize(); DCHECK_LE(used_size, size); DCHECK_GE(used_size, T::kHeaderSize); return VisitPartiallyWithSnapshot<T, TBodyDescriptor>(map, object, used_size, size); } template <typename T> int VisitEmbedderTracingSubclass(Map map, T object) { DCHECK(object.IsApiWrapper()); int size = VisitJSObjectSubclass(map, object); if (size && embedder_tracing_enabled_) { // Success: The object needs to be processed for embedder references on // the main thread. embedder_objects_.Push(object); } return size; } template <typename T> int VisitLeftTrimmableArray(Map map, T object) { // The synchronized_length() function checks that the length is a Smi. // This is not necessarily the case if the array is being left-trimmed. Object length = object.unchecked_synchronized_length(); if (!ShouldVisit(object)) return 0; // The cached length must be the actual length as the array is not black. // Left trimming marks the array black before over-writing the length. DCHECK(length.IsSmi()); int size = T::SizeFor(Smi::ToInt(length)); VisitMapPointer(object); T::BodyDescriptor::IterateBody(map, object, size, this); return size; } template <typename T> int VisitFullyWithSnapshot(Map map, T object) { using TBodyDescriptor = typename T::BodyDescriptor; int size = TBodyDescriptor::SizeOf(map, object); return VisitPartiallyWithSnapshot<T, TBodyDescriptor>(map, object, size, size); } template <typename T, typename TBodyDescriptor = typename T::BodyDescriptor> int VisitPartiallyWithSnapshot(Map map, T object, int used_size, int size) { const SlotSnapshot& snapshot = MakeSlotSnapshot<T, TBodyDescriptor>(map, object, used_size); if (!ShouldVisit(object)) return 0; VisitPointersInSnapshot(object, snapshot); return size; } template <typename T, typename TBodyDescriptor> const SlotSnapshot& MakeSlotSnapshot(Map map, T object, int size) { SlotSnapshottingVisitor visitor(&slot_snapshot_); visitor.VisitPointer(object, object.map_slot()); TBodyDescriptor::IterateBody(map, object, size, &visitor); return slot_snapshot_; } void RecordRelocSlot(Code host, RelocInfo* rinfo, HeapObject target) { MarkCompactCollector::RecordRelocSlotInfo info = MarkCompactCollector::PrepareRecordRelocSlot(host, rinfo, target); if (info.should_record) { MemoryChunkData& data = (*memory_chunk_data_)[info.memory_chunk]; if (!data.typed_slots) { data.typed_slots.reset(new TypedSlots()); } data.typed_slots->Insert(info.slot_type, info.offset); } } ConcurrentMarking::MarkingWorklist::View shared_; WeakObjects* weak_objects_; ConcurrentMarking::EmbedderTracingWorklist::View embedder_objects_; ConcurrentMarkingState marking_state_; MemoryChunkDataMap* memory_chunk_data_; int task_id_; SlotSnapshot slot_snapshot_; bool embedder_tracing_enabled_; const unsigned mark_compact_epoch_; bool is_forced_gc_; BytecodeFlushMode bytecode_flush_mode_; }; // Strings can change maps due to conversion to thin string or external strings. // Use unchecked cast to avoid data race in slow dchecks. template <> ConsString ConcurrentMarkingVisitor::Cast(HeapObject object) { return ConsString::unchecked_cast(object); } template <> SlicedString ConcurrentMarkingVisitor::Cast(HeapObject object) { return SlicedString::unchecked_cast(object); } template <> ThinString ConcurrentMarkingVisitor::Cast(HeapObject object) { return ThinString::unchecked_cast(object); } template <> SeqOneByteString ConcurrentMarkingVisitor::Cast(HeapObject object) { return SeqOneByteString::unchecked_cast(object); } template <> SeqTwoByteString ConcurrentMarkingVisitor::Cast(HeapObject object) { return SeqTwoByteString::unchecked_cast(object); } // Fixed array can become a free space during left trimming. template <> FixedArray ConcurrentMarkingVisitor::Cast(HeapObject object) { return FixedArray::unchecked_cast(object); } class ConcurrentMarking::Task : public CancelableTask { public: Task(Isolate* isolate, ConcurrentMarking* concurrent_marking, TaskState* task_state, int task_id) : CancelableTask(isolate), concurrent_marking_(concurrent_marking), task_state_(task_state), task_id_(task_id) {} ~Task() override = default; private: // v8::internal::CancelableTask overrides. void RunInternal() override { concurrent_marking_->Run(task_id_, task_state_); } ConcurrentMarking* concurrent_marking_; TaskState* task_state_; int task_id_; DISALLOW_COPY_AND_ASSIGN(Task); }; ConcurrentMarking::ConcurrentMarking(Heap* heap, MarkingWorklist* shared, MarkingWorklist* on_hold, WeakObjects* weak_objects, EmbedderTracingWorklist* embedder_objects) : heap_(heap), shared_(shared), on_hold_(on_hold), weak_objects_(weak_objects), embedder_objects_(embedder_objects) { // The runtime flag should be set only if the compile time flag was set. #ifndef V8_CONCURRENT_MARKING CHECK(!FLAG_concurrent_marking && !FLAG_parallel_marking); #endif } void ConcurrentMarking::Run(int task_id, TaskState* task_state) { TRACE_BACKGROUND_GC(heap_->tracer(), GCTracer::BackgroundScope::MC_BACKGROUND_MARKING); size_t kBytesUntilInterruptCheck = 64 * KB; int kObjectsUntilInterrupCheck = 1000; ConcurrentMarkingVisitor visitor( shared_, &task_state->memory_chunk_data, weak_objects_, embedder_objects_, task_id, heap_->local_embedder_heap_tracer()->InUse(), task_state->mark_compact_epoch, task_state->is_forced_gc); double time_ms; size_t marked_bytes = 0; if (FLAG_trace_concurrent_marking) { heap_->isolate()->PrintWithTimestamp( "Starting concurrent marking task %d\n", task_id); } bool ephemeron_marked = false; { TimedScope scope(&time_ms); { Ephemeron ephemeron; while (weak_objects_->current_ephemerons.Pop(task_id, &ephemeron)) { if (visitor.ProcessEphemeron(ephemeron.key, ephemeron.value)) { ephemeron_marked = true; } } } bool done = false; while (!done) { size_t current_marked_bytes = 0; int objects_processed = 0; while (current_marked_bytes < kBytesUntilInterruptCheck && objects_processed < kObjectsUntilInterrupCheck) { HeapObject object; if (!shared_->Pop(task_id, &object)) { done = true; break; } objects_processed++; // The order of the two loads is important. Address new_space_top = heap_->new_space()->original_top_acquire(); Address new_space_limit = heap_->new_space()->original_limit_relaxed(); Address new_large_object = heap_->new_lo_space()->pending_object(); Address addr = object.address(); if ((new_space_top <= addr && addr < new_space_limit) || addr == new_large_object) { on_hold_->Push(task_id, object); } else { Map map = object.synchronized_map(); current_marked_bytes += visitor.Visit(map, object); } } marked_bytes += current_marked_bytes; base::AsAtomicWord::Relaxed_Store<size_t>(&task_state->marked_bytes, marked_bytes); if (task_state->preemption_request) { TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.gc"), "ConcurrentMarking::Run Preempted"); break; } } if (done) { Ephemeron ephemeron; while (weak_objects_->discovered_ephemerons.Pop(task_id, &ephemeron)) { if (visitor.ProcessEphemeron(ephemeron.key, ephemeron.value)) { ephemeron_marked = true; } } } shared_->FlushToGlobal(task_id); on_hold_->FlushToGlobal(task_id); embedder_objects_->FlushToGlobal(task_id); weak_objects_->transition_arrays.FlushToGlobal(task_id); weak_objects_->ephemeron_hash_tables.FlushToGlobal(task_id); weak_objects_->current_ephemerons.FlushToGlobal(task_id); weak_objects_->next_ephemerons.FlushToGlobal(task_id); weak_objects_->discovered_ephemerons.FlushToGlobal(task_id); weak_objects_->weak_references.FlushToGlobal(task_id); weak_objects_->js_weak_refs.FlushToGlobal(task_id); weak_objects_->weak_cells.FlushToGlobal(task_id); weak_objects_->weak_objects_in_code.FlushToGlobal(task_id); weak_objects_->bytecode_flushing_candidates.FlushToGlobal(task_id); weak_objects_->flushed_js_functions.FlushToGlobal(task_id); base::AsAtomicWord::Relaxed_Store<size_t>(&task_state->marked_bytes, 0); total_marked_bytes_ += marked_bytes; if (ephemeron_marked) { set_ephemeron_marked(true); } { base::MutexGuard guard(&pending_lock_); is_pending_[task_id] = false; --pending_task_count_; pending_condition_.NotifyAll(); } } if (FLAG_trace_concurrent_marking) { heap_->isolate()->PrintWithTimestamp( "Task %d concurrently marked %dKB in %.2fms\n", task_id, static_cast<int>(marked_bytes / KB), time_ms); } } void ConcurrentMarking::ScheduleTasks() { DCHECK(FLAG_parallel_marking || FLAG_concurrent_marking); DCHECK(!heap_->IsTearingDown()); base::MutexGuard guard(&pending_lock_); DCHECK_EQ(0, pending_task_count_); if (task_count_ == 0) { static const int num_cores = V8::GetCurrentPlatform()->NumberOfWorkerThreads() + 1; #if defined(V8_OS_MACOSX) // Mac OSX 10.11 and prior seems to have trouble when doing concurrent // marking on competing hyper-threads (regresses Octane/Splay). As such, // only use num_cores/2, leaving one of those for the main thread. // TODO(ulan): Use all cores on Mac 10.12+. task_count_ = Max(1, Min(kMaxTasks, (num_cores / 2) - 1)); #else // defined(OS_MACOSX) // On other platforms use all logical cores, leaving one for the main // thread. task_count_ = Max(1, Min(kMaxTasks, num_cores - 1)); #endif // defined(OS_MACOSX) } // Task id 0 is for the main thread. for (int i = 1; i <= task_count_; i++) { if (!is_pending_[i]) { if (FLAG_trace_concurrent_marking) { heap_->isolate()->PrintWithTimestamp( "Scheduling concurrent marking task %d\n", i); } task_state_[i].preemption_request = false; task_state_[i].mark_compact_epoch = heap_->mark_compact_collector()->epoch(); task_state_[i].is_forced_gc = heap_->is_current_gc_forced(); is_pending_[i] = true; ++pending_task_count_; auto task = base::make_unique<Task>(heap_->isolate(), this, &task_state_[i], i); cancelable_id_[i] = task->id(); V8::GetCurrentPlatform()->CallOnWorkerThread(std::move(task)); } } DCHECK_EQ(task_count_, pending_task_count_); } void ConcurrentMarking::RescheduleTasksIfNeeded() { DCHECK(FLAG_parallel_marking || FLAG_concurrent_marking); if (heap_->IsTearingDown()) return; { base::MutexGuard guard(&pending_lock_); if (pending_task_count_ > 0) return; } if (!shared_->IsGlobalPoolEmpty() || !weak_objects_->current_ephemerons.IsEmpty() || !weak_objects_->discovered_ephemerons.IsEmpty()) { ScheduleTasks(); } } bool ConcurrentMarking::Stop(StopRequest stop_request) { DCHECK(FLAG_parallel_marking || FLAG_concurrent_marking); base::MutexGuard guard(&pending_lock_); if (pending_task_count_ == 0) return false; if (stop_request != StopRequest::COMPLETE_TASKS_FOR_TESTING) { CancelableTaskManager* task_manager = heap_->isolate()->cancelable_task_manager(); for (int i = 1; i <= task_count_; i++) { if (is_pending_[i]) { if (task_manager->TryAbort(cancelable_id_[i]) == TryAbortResult::kTaskAborted) { is_pending_[i] = false; --pending_task_count_; } else if (stop_request == StopRequest::PREEMPT_TASKS) { task_state_[i].preemption_request = true; } } } } while (pending_task_count_ > 0) { pending_condition_.Wait(&pending_lock_); } for (int i = 1; i <= task_count_; i++) { DCHECK(!is_pending_[i]); } return true; } bool ConcurrentMarking::IsStopped() { if (!FLAG_concurrent_marking) return true; base::MutexGuard guard(&pending_lock_); return pending_task_count_ == 0; } void ConcurrentMarking::FlushMemoryChunkData( MajorNonAtomicMarkingState* marking_state) { DCHECK_EQ(pending_task_count_, 0); for (int i = 1; i <= task_count_; i++) { MemoryChunkDataMap& memory_chunk_data = task_state_[i].memory_chunk_data; for (auto& pair : memory_chunk_data) { // ClearLiveness sets the live bytes to zero. // Pages with zero live bytes might be already unmapped. MemoryChunk* memory_chunk = pair.first; MemoryChunkData& data = pair.second; if (data.live_bytes) { marking_state->IncrementLiveBytes(memory_chunk, data.live_bytes); } if (data.typed_slots) { RememberedSet<OLD_TO_OLD>::MergeTyped(memory_chunk, std::move(data.typed_slots)); } } memory_chunk_data.clear(); task_state_[i].marked_bytes = 0; } total_marked_bytes_ = 0; } void ConcurrentMarking::ClearMemoryChunkData(MemoryChunk* chunk) { for (int i = 1; i <= task_count_; i++) { auto it = task_state_[i].memory_chunk_data.find(chunk); if (it != task_state_[i].memory_chunk_data.end()) { it->second.live_bytes = 0; it->second.typed_slots.reset(); } } } size_t ConcurrentMarking::TotalMarkedBytes() { size_t result = 0; for (int i = 1; i <= task_count_; i++) { result += base::AsAtomicWord::Relaxed_Load<size_t>(&task_state_[i].marked_bytes); } result += total_marked_bytes_; return result; } ConcurrentMarking::PauseScope::PauseScope(ConcurrentMarking* concurrent_marking) : concurrent_marking_(concurrent_marking), resume_on_exit_(FLAG_concurrent_marking && concurrent_marking_->Stop( ConcurrentMarking::StopRequest::PREEMPT_TASKS)) { DCHECK_IMPLIES(resume_on_exit_, FLAG_concurrent_marking); } ConcurrentMarking::PauseScope::~PauseScope() { if (resume_on_exit_) concurrent_marking_->RescheduleTasksIfNeeded(); } } // namespace internal } // namespace v8