path: root/deps/v8/src/heap/concurrent-marking.cc

// Copyright 2017 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "src/heap/concurrent-marking.h"

#include <stack>
#include <unordered_map>

#include "include/v8config.h"
#include "src/execution/isolate.h"
#include "src/heap/gc-tracer.h"
#include "src/heap/heap-inl.h"
#include "src/heap/heap.h"
#include "src/heap/mark-compact-inl.h"
#include "src/heap/mark-compact.h"
#include "src/heap/marking.h"
#include "src/heap/objects-visiting-inl.h"
#include "src/heap/objects-visiting.h"
#include "src/heap/worklist.h"
#include "src/init/v8.h"
#include "src/objects/data-handler-inl.h"
#include "src/objects/embedder-data-array-inl.h"
#include "src/objects/hash-table-inl.h"
#include "src/objects/slots-inl.h"
#include "src/objects/transitions-inl.h"
#include "src/utils/utils-inl.h"
#include "src/utils/utils.h"

namespace v8 {
namespace internal {

class ConcurrentMarkingState final
    : public MarkingStateBase<ConcurrentMarkingState, AccessMode::ATOMIC> {
 public:
  explicit ConcurrentMarkingState(MemoryChunkDataMap* memory_chunk_data)
      : memory_chunk_data_(memory_chunk_data) {}

  ConcurrentBitmap<AccessMode::ATOMIC>* bitmap(const MemoryChunk* chunk) {
    DCHECK_EQ(reinterpret_cast<intptr_t>(&chunk->marking_bitmap_) -
                  reinterpret_cast<intptr_t>(chunk),
              MemoryChunk::kMarkBitmapOffset);
    return chunk->marking_bitmap<AccessMode::ATOMIC>();
  }

  void IncrementLiveBytes(MemoryChunk* chunk, intptr_t by) {
    (*memory_chunk_data_)[chunk].live_bytes += by;
  }

  // The live_bytes and SetLiveBytes methods of the marking state are
  // not used by the concurrent marker.

 private:
  MemoryChunkDataMap* memory_chunk_data_;
};

// Helper class for storing in-object slot addresses and values.
class SlotSnapshot {
 public:
  SlotSnapshot() : number_of_slots_(0) {}
  int number_of_slots() const { return number_of_slots_; }
  ObjectSlot slot(int i) const { return snapshot_[i].first; }
  Object value(int i) const { return snapshot_[i].second; }
  void clear() { number_of_slots_ = 0; }
  void add(ObjectSlot slot, Object value) {
    snapshot_[number_of_slots_++] = {slot, value};
  }

 private:
  static const int kMaxSnapshotSize = JSObject::kMaxInstanceSize / kTaggedSize;
  int number_of_slots_;
  std::pair<ObjectSlot, Object> snapshot_[kMaxSnapshotSize];
  DISALLOW_COPY_AND_ASSIGN(SlotSnapshot);
};

class ConcurrentMarkingVisitor final
    : public HeapVisitor<int, ConcurrentMarkingVisitor> {
 public:
  using BaseClass = HeapVisitor<int, ConcurrentMarkingVisitor>;

  explicit ConcurrentMarkingVisitor(
      ConcurrentMarking::MarkingWorklist* shared,
      MemoryChunkDataMap* memory_chunk_data, WeakObjects* weak_objects,
      ConcurrentMarking::EmbedderTracingWorklist* embedder_objects, int task_id,
      bool embedder_tracing_enabled, unsigned mark_compact_epoch,
      bool is_forced_gc)
      : shared_(shared, task_id),
        weak_objects_(weak_objects),
        embedder_objects_(embedder_objects, task_id),
        marking_state_(memory_chunk_data),
        memory_chunk_data_(memory_chunk_data),
        task_id_(task_id),
        embedder_tracing_enabled_(embedder_tracing_enabled),
        mark_compact_epoch_(mark_compact_epoch),
        is_forced_gc_(is_forced_gc) {
    // It is not safe to access flags from concurrent marking visitor. So
    // set the bytecode flush mode based on the flags here
    bytecode_flush_mode_ = Heap::GetBytecodeFlushMode();
  }

  template <typename T>
  static V8_INLINE T Cast(HeapObject object) {
    return T::cast(object);
  }

  bool ShouldVisit(HeapObject object) {
    return marking_state_.GreyToBlack(object);
  }

  bool AllowDefaultJSObjectVisit() { return false; }

  template <typename THeapObjectSlot>
  void ProcessStrongHeapObject(HeapObject host, THeapObjectSlot slot,
                               HeapObject heap_object) {
    MarkObject(heap_object);
    MarkCompactCollector::RecordSlot(host, slot, heap_object);
  }

  template <typename THeapObjectSlot>
  void ProcessWeakHeapObject(HeapObject host, THeapObjectSlot slot,
                             HeapObject heap_object) {
#ifdef THREAD_SANITIZER
    MemoryChunk::FromHeapObject(heap_object)->SynchronizedHeapLoad();
#endif
    if (marking_state_.IsBlackOrGrey(heap_object)) {
      // Weak references with live values are directly processed here to
      // reduce the processing time of weak cells during the main GC
      // pause.
      MarkCompactCollector::RecordSlot(host, slot, heap_object);
    } else {
      // If we do not know about liveness of the value, we have to process
      // the reference when we know the liveness of the whole transitive
      // closure.
      weak_objects_->weak_references.Push(task_id_, std::make_pair(host, slot));
    }
  }

  void VisitPointers(HeapObject host, ObjectSlot start,
                     ObjectSlot end) override {
    VisitPointersImpl(host, start, end);
  }

  void VisitPointers(HeapObject host, MaybeObjectSlot start,
                     MaybeObjectSlot end) override {
    VisitPointersImpl(host, start, end);
  }

  template <typename TSlot>
  V8_INLINE void VisitPointersImpl(HeapObject host, TSlot start, TSlot end) {
    using THeapObjectSlot = typename TSlot::THeapObjectSlot;
    for (TSlot slot = start; slot < end; ++slot) {
      typename TSlot::TObject object = slot.Relaxed_Load();
      HeapObject heap_object;
      if (object.GetHeapObjectIfStrong(&heap_object)) {
        // If the reference changes concurrently from strong to weak, the write
        // barrier will treat the weak reference as strong, so we won't miss the
        // weak reference.
        ProcessStrongHeapObject(host, THeapObjectSlot(slot), heap_object);
      } else if (TSlot::kCanBeWeak &&
                 object.GetHeapObjectIfWeak(&heap_object)) {
        ProcessWeakHeapObject(host, THeapObjectSlot(slot), heap_object);
      }
    }
  }

  // Weak list pointers should be ignored during marking. The lists are
  // reconstructed after GC.
  void VisitCustomWeakPointers(HeapObject host, ObjectSlot start,
                               ObjectSlot end) final {}

  void VisitEmbeddedPointer(Code host, RelocInfo* rinfo) final {
    DCHECK(RelocInfo::IsEmbeddedObjectMode(rinfo->rmode()));
    HeapObject object = rinfo->target_object();
    RecordRelocSlot(host, rinfo, object);
    if (!marking_state_.IsBlackOrGrey(object)) {
      if (host.IsWeakObject(object)) {
        weak_objects_->weak_objects_in_code.Push(task_id_,
                                                 std::make_pair(object, host));
      } else {
        MarkObject(object);
      }
    }
  }

  void VisitCodeTarget(Code host, RelocInfo* rinfo) final {
    DCHECK(RelocInfo::IsCodeTargetMode(rinfo->rmode()));
    Code target = Code::GetCodeFromTargetAddress(rinfo->target_address());
    RecordRelocSlot(host, rinfo, target);
    MarkObject(target);
  }

  void VisitPointersInSnapshot(HeapObject host, const SlotSnapshot& snapshot) {
    for (int i = 0; i < snapshot.number_of_slots(); i++) {
      ObjectSlot slot = snapshot.slot(i);
      Object object = snapshot.value(i);
      DCHECK(!HasWeakHeapObjectTag(object));
      if (!object.IsHeapObject()) continue;
      HeapObject heap_object = HeapObject::cast(object);
      MarkObject(heap_object);
      MarkCompactCollector::RecordSlot(host, slot, heap_object);
    }
  }

  // ===========================================================================
  // JS object =================================================================
  // ===========================================================================

  int VisitJSObject(Map map, JSObject object) {
    return VisitJSObjectSubclass(map, object);
  }

  int VisitJSObjectFast(Map map, JSObject object) {
    return VisitJSObjectSubclassFast(map, object);
  }

  int VisitWasmInstanceObject(Map map, WasmInstanceObject object) {
    return VisitJSObjectSubclass(map, object);
  }

  int VisitJSWeakRef(Map map, JSWeakRef weak_ref) {
    int size = VisitJSObjectSubclass(map, weak_ref);
    if (size == 0) {
      return 0;
    }
    if (weak_ref.target().IsHeapObject()) {
      HeapObject target = HeapObject::cast(weak_ref.target());
#ifdef THREAD_SANITIZER
      MemoryChunk::FromHeapObject(target)->SynchronizedHeapLoad();
#endif
      if (marking_state_.IsBlackOrGrey(target)) {
        // Record the slot inside the JSWeakRef, since the
        // VisitJSObjectSubclass above didn't visit it.
        ObjectSlot slot = weak_ref.RawField(JSWeakRef::kTargetOffset);
        MarkCompactCollector::RecordSlot(weak_ref, slot, target);
      } else {
        // JSWeakRef points to a potentially dead object. We have to process
        // them when we know the liveness of the whole transitive closure.
        weak_objects_->js_weak_refs.Push(task_id_, weak_ref);
      }
    }
    return size;
  }

  int VisitWeakCell(Map map, WeakCell weak_cell) {
    if (!ShouldVisit(weak_cell)) return 0;

    int size = WeakCell::BodyDescriptor::SizeOf(map, weak_cell);
    VisitMapPointer(weak_cell);
    WeakCell::BodyDescriptor::IterateBody(map, weak_cell, size, this);
    if (weak_cell.target().IsHeapObject()) {
      HeapObject target = HeapObject::cast(weak_cell.target());
#ifdef THREAD_SANITIZER
      MemoryChunk::FromHeapObject(target)->SynchronizedHeapLoad();
#endif
      if (marking_state_.IsBlackOrGrey(target)) {
        // Record the slot inside the WeakCell, since the IterateBody above
        // didn't visit it.
        ObjectSlot slot = weak_cell.RawField(WeakCell::kTargetOffset);
        MarkCompactCollector::RecordSlot(weak_cell, slot, target);
      } else {
        // WeakCell points to a potentially dead object. We have to process
        // them when we know the liveness of the whole transitive closure.
        weak_objects_->weak_cells.Push(task_id_, weak_cell);
      }
    }
    return size;
  }

  // Some JS objects can carry back links to embedders that contain information
  // relevant to the garbage collectors.

  int VisitJSApiObject(Map map, JSObject object) {
    return VisitEmbedderTracingSubclass(map, object);
  }

  int VisitJSArrayBuffer(Map map, JSArrayBuffer object) {
    return VisitEmbedderTracingSubclass(map, object);
  }

  int VisitJSDataView(Map map, JSDataView object) {
    return VisitEmbedderTracingSubclass(map, object);
  }

  int VisitJSTypedArray(Map map, JSTypedArray object) {
    return VisitEmbedderTracingSubclass(map, object);
  }

  // ===========================================================================
  // Strings with pointers =====================================================
  // ===========================================================================

  int VisitConsString(Map map, ConsString object) {
    return VisitFullyWithSnapshot(map, object);
  }

  int VisitSlicedString(Map map, SlicedString object) {
    return VisitFullyWithSnapshot(map, object);
  }

  int VisitThinString(Map map, ThinString object) {
    return VisitFullyWithSnapshot(map, object);
  }

  // ===========================================================================
  // Strings without pointers ==================================================
  // ===========================================================================

  int VisitSeqOneByteString(Map map, SeqOneByteString object) {
    if (!ShouldVisit(object)) return 0;
    VisitMapPointer(object);
    return SeqOneByteString::SizeFor(object.synchronized_length());
  }

  int VisitSeqTwoByteString(Map map, SeqTwoByteString object) {
    if (!ShouldVisit(object)) return 0;
    VisitMapPointer(object);
    return SeqTwoByteString::SizeFor(object.synchronized_length());
  }

  // ===========================================================================
  // Fixed array object ========================================================
  // ===========================================================================

  int VisitFixedArrayWithProgressBar(Map map, FixedArray object,
                                     MemoryChunk* chunk) {
    // The concurrent marker can process larger chunks than the main thread
    // marker.
    const int kProgressBarScanningChunk =
        RoundUp(kMaxRegularHeapObjectSize, kTaggedSize);
    DCHECK(marking_state_.IsBlackOrGrey(object));
    marking_state_.GreyToBlack(object);
    int size = FixedArray::BodyDescriptor::SizeOf(map, object);
    size_t current_progress_bar = chunk->ProgressBar();
    int start = static_cast<int>(current_progress_bar);
    if (start == 0) start = FixedArray::BodyDescriptor::kStartOffset;
    int end = Min(size, start + kProgressBarScanningChunk);
    if (start < end) {
      VisitPointers(object, object.RawField(start), object.RawField(end));
      bool success = chunk->TrySetProgressBar(current_progress_bar, end);
      CHECK(success);
      if (end < size) {
        // The object can be pushed back onto the marking worklist only after
        // progress bar was updated.
        shared_.Push(object);
      }
    }
    return end - start;
  }

  int VisitFixedArray(Map map, FixedArray object) {
    // Arrays with the progress bar are not left-trimmable because they reside
    // in the large object space.
    MemoryChunk* chunk = MemoryChunk::FromHeapObject(object);
    return chunk->IsFlagSet<AccessMode::ATOMIC>(MemoryChunk::HAS_PROGRESS_BAR)
               ? VisitFixedArrayWithProgressBar(map, object, chunk)
               : VisitLeftTrimmableArray(map, object);
  }

  int VisitFixedDoubleArray(Map map, FixedDoubleArray object) {
    return VisitLeftTrimmableArray(map, object);
  }

  // ===========================================================================
  // Side-effectful visitation.
  // ===========================================================================

  int VisitSharedFunctionInfo(Map map, SharedFunctionInfo shared_info) {
    if (!ShouldVisit(shared_info)) return 0;

    int size = SharedFunctionInfo::BodyDescriptor::SizeOf(map, shared_info);
    VisitMapPointer(shared_info);
    SharedFunctionInfo::BodyDescriptor::IterateBody(map, shared_info, size,
                                                    this);

    // If the SharedFunctionInfo has old bytecode, mark it as flushable,
    // otherwise visit the function data field strongly.
    if (shared_info.ShouldFlushBytecode(bytecode_flush_mode_)) {
      weak_objects_->bytecode_flushing_candidates.Push(task_id_, shared_info);
    } else {
      VisitPointer(shared_info, shared_info.RawField(
                                    SharedFunctionInfo::kFunctionDataOffset));
    }
    return size;
  }

  int VisitBytecodeArray(Map map, BytecodeArray object) {
    if (!ShouldVisit(object)) return 0;
    int size = BytecodeArray::BodyDescriptor::SizeOf(map, object);
    VisitMapPointer(object);
    BytecodeArray::BodyDescriptor::IterateBody(map, object, size, this);
    if (!is_forced_gc_) {
      object.MakeOlder();
    }
    return size;
  }

  int VisitJSFunction(Map map, JSFunction object) {
    int size = VisitJSObjectSubclass(map, object);

    // Check if the JSFunction needs reset due to bytecode being flushed.
    if (bytecode_flush_mode_ != BytecodeFlushMode::kDoNotFlushBytecode &&
        object.NeedsResetDueToFlushedBytecode()) {
      weak_objects_->flushed_js_functions.Push(task_id_, object);
    }

    return size;
  }

  int VisitMap(Map meta_map, Map map) {
    if (!ShouldVisit(map)) return 0;
    int size = Map::BodyDescriptor::SizeOf(meta_map, map);
    if (map.CanTransition()) {
      // Maps that can transition share their descriptor arrays and require
      // special visiting logic to avoid memory leaks.
      // Since descriptor arrays are potentially shared, ensure that only the
      // descriptors that belong to this map are marked. The first time a
      // non-empty descriptor array is marked, its header is also visited. The
      // slot holding the descriptor array will be implicitly recorded when the
      // pointer fields of this map are visited.
      DescriptorArray descriptors = map.synchronized_instance_descriptors();
      MarkDescriptorArrayBlack(descriptors);
      int number_of_own_descriptors = map.NumberOfOwnDescriptors();
      if (number_of_own_descriptors) {
        // It is possible that the concurrent marker observes the
        // number_of_own_descriptors out of sync with the descriptors. In that
        // case the marking write barrier for the descriptor array will ensure
        // that all required descriptors are marked. The concurrent marker
        // just should avoid crashing in that case. That's why we need the
        // std::min<int>() below.
        VisitDescriptors(descriptors,
                         std::min<int>(number_of_own_descriptors,
                                       descriptors.number_of_descriptors()));
      }
      // Mark the pointer fields of the Map. Since the transitions array has
      // been marked already, it is fine that one of these fields contains a
      // pointer to it.
    }
    Map::BodyDescriptor::IterateBody(meta_map, map, size, this);
    return size;
  }

  void VisitDescriptors(DescriptorArray descriptor_array,
                        int number_of_own_descriptors) {
    int16_t new_marked = static_cast<int16_t>(number_of_own_descriptors);
    int16_t old_marked = descriptor_array.UpdateNumberOfMarkedDescriptors(
        mark_compact_epoch_, new_marked);
    if (old_marked < new_marked) {
      VisitPointers(
          descriptor_array,
          MaybeObjectSlot(descriptor_array.GetDescriptorSlot(old_marked)),
          MaybeObjectSlot(descriptor_array.GetDescriptorSlot(new_marked)));
    }
  }

  int VisitDescriptorArray(Map map, DescriptorArray array) {
    if (!ShouldVisit(array)) return 0;
    VisitMapPointer(array);
    int size = DescriptorArray::BodyDescriptor::SizeOf(map, array);
    VisitPointers(array, array.GetFirstPointerSlot(),
                  array.GetDescriptorSlot(0));
    VisitDescriptors(array, array.number_of_descriptors());
    return size;
  }

  int VisitTransitionArray(Map map, TransitionArray array) {
    if (!ShouldVisit(array)) return 0;
    VisitMapPointer(array);
    int size = TransitionArray::BodyDescriptor::SizeOf(map, array);
    TransitionArray::BodyDescriptor::IterateBody(map, array, size, this);
    weak_objects_->transition_arrays.Push(task_id_, array);
    return size;
  }

  int VisitJSWeakCollection(Map map, JSWeakCollection object) {
    return VisitJSObjectSubclass(map, object);
  }

  int VisitEphemeronHashTable(Map map, EphemeronHashTable table) {
    if (!ShouldVisit(table)) return 0;
    weak_objects_->ephemeron_hash_tables.Push(task_id_, table);

    for (int i = 0; i < table.Capacity(); i++) {
      ObjectSlot key_slot =
          table.RawFieldOfElementAt(EphemeronHashTable::EntryToIndex(i));
      HeapObject key = HeapObject::cast(table.KeyAt(i));
#ifdef THREAD_SANITIZER
      MemoryChunk::FromHeapObject(key)->SynchronizedHeapLoad();
#endif
      MarkCompactCollector::RecordSlot(table, key_slot, key);

      ObjectSlot value_slot =
          table.RawFieldOfElementAt(EphemeronHashTable::EntryToValueIndex(i));

      if (marking_state_.IsBlackOrGrey(key)) {
        VisitPointer(table, value_slot);

      } else {
        Object value_obj = table.ValueAt(i);

        if (value_obj.IsHeapObject()) {
          HeapObject value = HeapObject::cast(value_obj);
#ifdef THREAD_SANITIZER
          MemoryChunk::FromHeapObject(value)->SynchronizedHeapLoad();
#endif
          MarkCompactCollector::RecordSlot(table, value_slot, value);

          // Revisit ephemerons with both key and value unreachable at end
          // of concurrent marking cycle.
          if (marking_state_.IsWhite(value)) {
            weak_objects_->discovered_ephemerons.Push(task_id_,
                                                      Ephemeron{key, value});
          }
        }
      }
    }

    return table.SizeFromMap(map);
  }

  // Implements ephemeron semantics: Marks value if key is already reachable.
  // Returns true if value was actually marked.
  bool ProcessEphemeron(HeapObject key, HeapObject value) {
    if (marking_state_.IsBlackOrGrey(key)) {
      if (marking_state_.WhiteToGrey(value)) {
        shared_.Push(value);
        return true;
      }

    } else if (marking_state_.IsWhite(value)) {
      weak_objects_->next_ephemerons.Push(task_id_, Ephemeron{key, value});
    }

    return false;
  }

  void MarkObject(HeapObject object) {
#ifdef THREAD_SANITIZER
    MemoryChunk::FromHeapObject(object)->SynchronizedHeapLoad();
#endif
    if (marking_state_.WhiteToGrey(object)) {
      shared_.Push(object);
    }
  }

  void MarkDescriptorArrayBlack(DescriptorArray descriptors) {
    marking_state_.WhiteToGrey(descriptors);
    if (marking_state_.GreyToBlack(descriptors)) {
      VisitPointers(descriptors, descriptors.GetFirstPointerSlot(),
                    descriptors.GetDescriptorSlot(0));
    }
  }

 private:
  // Helper class for collecting in-object slot addresses and values.
  class SlotSnapshottingVisitor final : public ObjectVisitor {
   public:
    explicit SlotSnapshottingVisitor(SlotSnapshot* slot_snapshot)
        : slot_snapshot_(slot_snapshot) {
      slot_snapshot_->clear();
    }

    void VisitPointers(HeapObject host, ObjectSlot start,
                       ObjectSlot end) override {
      for (ObjectSlot p = start; p < end; ++p) {
        Object object = p.Relaxed_Load();
        slot_snapshot_->add(p, object);
      }
    }

    void VisitPointers(HeapObject host, MaybeObjectSlot start,
                       MaybeObjectSlot end) override {
      // This should never happen, because we don't use snapshotting for objects
      // which contain weak references.
      UNREACHABLE();
    }

    void VisitCodeTarget(Code host, RelocInfo* rinfo) final {
      // This should never happen, because snapshotting is performed only on
      // JSObjects (and derived classes).
      UNREACHABLE();
    }

    void VisitEmbeddedPointer(Code host, RelocInfo* rinfo) final {
      // This should never happen, because snapshotting is performed only on
      // JSObjects (and derived classes).
      UNREACHABLE();
    }

    void VisitCustomWeakPointers(HeapObject host, ObjectSlot start,
                                 ObjectSlot end) override {
      DCHECK(host.IsWeakCell() || host.IsJSWeakRef());
    }

   private:
    SlotSnapshot* slot_snapshot_;
  };

  template <typename T>
  int VisitJSObjectSubclassFast(Map map, T object) {
    DCHECK_IMPLIES(FLAG_unbox_double_fields, map.HasFastPointerLayout());
    using TBodyDescriptor = typename T::FastBodyDescriptor;
    return VisitJSObjectSubclass<T, TBodyDescriptor>(map, object);
  }

  template <typename T, typename TBodyDescriptor = typename T::BodyDescriptor>
  int VisitJSObjectSubclass(Map map, T object) {
    int size = TBodyDescriptor::SizeOf(map, object);
    int used_size = map.UsedInstanceSize();
    DCHECK_LE(used_size, size);
    DCHECK_GE(used_size, T::kHeaderSize);
    return VisitPartiallyWithSnapshot<T, TBodyDescriptor>(map, object,
                                                          used_size, size);
  }

  template <typename T>
  int VisitEmbedderTracingSubclass(Map map, T object) {
    DCHECK(object.IsApiWrapper());
    int size = VisitJSObjectSubclass(map, object);
    if (size && embedder_tracing_enabled_) {
      // Success: The object needs to be processed for embedder references on
      // the main thread.
      embedder_objects_.Push(object);
    }
    return size;
  }

  template <typename T>
  int VisitLeftTrimmableArray(Map map, T object) {
    // The synchronized_length() function checks that the length is a Smi.
    // This is not necessarily the case if the array is being left-trimmed.
    Object length = object.unchecked_synchronized_length();
    if (!ShouldVisit(object)) return 0;
    // The cached length must be the actual length as the array is not black.
    // Left trimming marks the array black before over-writing the length.
    DCHECK(length.IsSmi());
    int size = T::SizeFor(Smi::ToInt(length));
    VisitMapPointer(object);
    T::BodyDescriptor::IterateBody(map, object, size, this);
    return size;
  }

  template <typename T>
  int VisitFullyWithSnapshot(Map map, T object) {
    using TBodyDescriptor = typename T::BodyDescriptor;
    int size = TBodyDescriptor::SizeOf(map, object);
    return VisitPartiallyWithSnapshot<T, TBodyDescriptor>(map, object, size,
                                                          size);
  }

  template <typename T, typename TBodyDescriptor = typename T::BodyDescriptor>
  int VisitPartiallyWithSnapshot(Map map, T object, int used_size, int size) {
    const SlotSnapshot& snapshot =
        MakeSlotSnapshot<T, TBodyDescriptor>(map, object, used_size);
    if (!ShouldVisit(object)) return 0;
    VisitPointersInSnapshot(object, snapshot);
    return size;
  }

  template <typename T, typename TBodyDescriptor>
  const SlotSnapshot& MakeSlotSnapshot(Map map, T object, int size) {
    SlotSnapshottingVisitor visitor(&slot_snapshot_);
    visitor.VisitPointer(object, object.map_slot());
    TBodyDescriptor::IterateBody(map, object, size, &visitor);
    return slot_snapshot_;
  }

  void RecordRelocSlot(Code host, RelocInfo* rinfo, HeapObject target) {
    MarkCompactCollector::RecordRelocSlotInfo info =
        MarkCompactCollector::PrepareRecordRelocSlot(host, rinfo, target);
    if (info.should_record) {
      MemoryChunkData& data = (*memory_chunk_data_)[info.memory_chunk];
      if (!data.typed_slots) {
        data.typed_slots.reset(new TypedSlots());
      }
      data.typed_slots->Insert(info.slot_type, info.offset);
    }
  }

  ConcurrentMarking::MarkingWorklist::View shared_;
  WeakObjects* weak_objects_;
  ConcurrentMarking::EmbedderTracingWorklist::View embedder_objects_;
  ConcurrentMarkingState marking_state_;
  MemoryChunkDataMap* memory_chunk_data_;
  int task_id_;
  SlotSnapshot slot_snapshot_;
  bool embedder_tracing_enabled_;
  const unsigned mark_compact_epoch_;
  bool is_forced_gc_;
  BytecodeFlushMode bytecode_flush_mode_;
};

// Strings can change maps due to conversion to thin string or external strings.
// Use unchecked cast to avoid data race in slow dchecks.
template <>
ConsString ConcurrentMarkingVisitor::Cast(HeapObject object) {
  return ConsString::unchecked_cast(object);
}

template <>
SlicedString ConcurrentMarkingVisitor::Cast(HeapObject object) {
  return SlicedString::unchecked_cast(object);
}

template <>
ThinString ConcurrentMarkingVisitor::Cast(HeapObject object) {
  return ThinString::unchecked_cast(object);
}

template <>
SeqOneByteString ConcurrentMarkingVisitor::Cast(HeapObject object) {
  return SeqOneByteString::unchecked_cast(object);
}

template <>
SeqTwoByteString ConcurrentMarkingVisitor::Cast(HeapObject object) {
  return SeqTwoByteString::unchecked_cast(object);
}

// Fixed array can become a free space during left trimming.
template <>
FixedArray ConcurrentMarkingVisitor::Cast(HeapObject object) {
  return FixedArray::unchecked_cast(object);
}

class ConcurrentMarking::Task : public CancelableTask {
 public:
  Task(Isolate* isolate, ConcurrentMarking* concurrent_marking,
       TaskState* task_state, int task_id)
      : CancelableTask(isolate),
        concurrent_marking_(concurrent_marking),
        task_state_(task_state),
        task_id_(task_id) {}

  ~Task() override = default;

 private:
  // v8::internal::CancelableTask overrides.
  void RunInternal() override {
    concurrent_marking_->Run(task_id_, task_state_);
  }

  ConcurrentMarking* concurrent_marking_;
  TaskState* task_state_;
  int task_id_;
  DISALLOW_COPY_AND_ASSIGN(Task);
};

ConcurrentMarking::ConcurrentMarking(Heap* heap, MarkingWorklist* shared,
                                     MarkingWorklist* on_hold,
                                     WeakObjects* weak_objects,
                                     EmbedderTracingWorklist* embedder_objects)
    : heap_(heap),
      shared_(shared),
      on_hold_(on_hold),
      weak_objects_(weak_objects),
      embedder_objects_(embedder_objects) {
// The runtime flag should be set only if the compile time flag was set.
#ifndef V8_CONCURRENT_MARKING
  CHECK(!FLAG_concurrent_marking && !FLAG_parallel_marking);
#endif
}

void ConcurrentMarking::Run(int task_id, TaskState* task_state) {
  TRACE_BACKGROUND_GC(heap_->tracer(),
                      GCTracer::BackgroundScope::MC_BACKGROUND_MARKING);
  size_t kBytesUntilInterruptCheck = 64 * KB;
  int kObjectsUntilInterrupCheck = 1000;
  ConcurrentMarkingVisitor visitor(
      shared_, &task_state->memory_chunk_data, weak_objects_, embedder_objects_,
      task_id, heap_->local_embedder_heap_tracer()->InUse(),
      task_state->mark_compact_epoch, task_state->is_forced_gc);
  double time_ms;
  size_t marked_bytes = 0;
  if (FLAG_trace_concurrent_marking) {
    heap_->isolate()->PrintWithTimestamp(
        "Starting concurrent marking task %d\n", task_id);
  }
  bool ephemeron_marked = false;

  {
    TimedScope scope(&time_ms);

    {
      Ephemeron ephemeron;

      while (weak_objects_->current_ephemerons.Pop(task_id, &ephemeron)) {
        if (visitor.ProcessEphemeron(ephemeron.key, ephemeron.value)) {
          ephemeron_marked = true;
        }
      }
    }

    bool done = false;
    while (!done) {
      size_t current_marked_bytes = 0;
      int objects_processed = 0;
      while (current_marked_bytes < kBytesUntilInterruptCheck &&
             objects_processed < kObjectsUntilInterrupCheck) {
        HeapObject object;
        if (!shared_->Pop(task_id, &object)) {
          done = true;
          break;
        }
        objects_processed++;
        // The order of the two loads is important.
        Address new_space_top = heap_->new_space()->original_top_acquire();
        Address new_space_limit = heap_->new_space()->original_limit_relaxed();
        Address new_large_object = heap_->new_lo_space()->pending_object();
        Address addr = object.address();
        if ((new_space_top <= addr && addr < new_space_limit) ||
            addr == new_large_object) {
          on_hold_->Push(task_id, object);
        } else {
          Map map = object.synchronized_map();
          current_marked_bytes += visitor.Visit(map, object);
        }
      }
      marked_bytes += current_marked_bytes;
      base::AsAtomicWord::Relaxed_Store<size_t>(&task_state->marked_bytes,
                                                marked_bytes);
      if (task_state->preemption_request) {
        TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.gc"),
                     "ConcurrentMarking::Run Preempted");
        break;
      }
    }

    if (done) {
      Ephemeron ephemeron;

      while (weak_objects_->discovered_ephemerons.Pop(task_id, &ephemeron)) {
        if (visitor.ProcessEphemeron(ephemeron.key, ephemeron.value)) {
          ephemeron_marked = true;
        }
      }
    }

    shared_->FlushToGlobal(task_id);
    on_hold_->FlushToGlobal(task_id);
    embedder_objects_->FlushToGlobal(task_id);

    weak_objects_->transition_arrays.FlushToGlobal(task_id);
    weak_objects_->ephemeron_hash_tables.FlushToGlobal(task_id);
    weak_objects_->current_ephemerons.FlushToGlobal(task_id);
    weak_objects_->next_ephemerons.FlushToGlobal(task_id);
    weak_objects_->discovered_ephemerons.FlushToGlobal(task_id);
    weak_objects_->weak_references.FlushToGlobal(task_id);
    weak_objects_->js_weak_refs.FlushToGlobal(task_id);
    weak_objects_->weak_cells.FlushToGlobal(task_id);
    weak_objects_->weak_objects_in_code.FlushToGlobal(task_id);
    weak_objects_->bytecode_flushing_candidates.FlushToGlobal(task_id);
    weak_objects_->flushed_js_functions.FlushToGlobal(task_id);
    base::AsAtomicWord::Relaxed_Store<size_t>(&task_state->marked_bytes, 0);
    total_marked_bytes_ += marked_bytes;

    if (ephemeron_marked) {
      set_ephemeron_marked(true);
    }

    {
      base::MutexGuard guard(&pending_lock_);
      is_pending_[task_id] = false;
      --pending_task_count_;
      pending_condition_.NotifyAll();
    }
  }
  if (FLAG_trace_concurrent_marking) {
    heap_->isolate()->PrintWithTimestamp(
        "Task %d concurrently marked %dKB in %.2fms\n", task_id,
        static_cast<int>(marked_bytes / KB), time_ms);
  }
}

void ConcurrentMarking::ScheduleTasks() {
  DCHECK(FLAG_parallel_marking || FLAG_concurrent_marking);
  DCHECK(!heap_->IsTearingDown());
  base::MutexGuard guard(&pending_lock_);
  if (total_task_count_ == 0) {
    static const int num_cores =
        V8::GetCurrentPlatform()->NumberOfWorkerThreads() + 1;
#if defined(V8_OS_MACOSX)
    // Mac OSX 10.11 and prior seems to have trouble when doing concurrent
    // marking on competing hyper-threads (regresses Octane/Splay). As such,
    // only use num_cores/2, leaving one of those for the main thread.
    // TODO(ulan): Use all cores on Mac 10.12+.
    total_task_count_ = Max(1, Min(kMaxTasks, (num_cores / 2) - 1));
#else   // defined(OS_MACOSX)
    // On other platforms use all logical cores, leaving one for the main
    // thread.
    total_task_count_ = Max(1, Min(kMaxTasks, num_cores - 1));
#endif  // defined(OS_MACOSX)
    DCHECK_LE(total_task_count_, kMaxTasks);
    // One task is for the main thread.
    STATIC_ASSERT(kMaxTasks + 1 <= MarkingWorklist::kMaxNumTasks);
  }
  // Task id 0 is for the main thread.
  for (int i = 1; i <= total_task_count_; i++) {
    if (!is_pending_[i]) {
      if (FLAG_trace_concurrent_marking) {
        heap_->isolate()->PrintWithTimestamp(
            "Scheduling concurrent marking task %d\n", i);
      }
      task_state_[i].preemption_request = false;
      task_state_[i].mark_compact_epoch =
          heap_->mark_compact_collector()->epoch();
      task_state_[i].is_forced_gc = heap_->is_current_gc_forced();
      is_pending_[i] = true;
      ++pending_task_count_;
      auto task =
          std::make_unique<Task>(heap_->isolate(), this, &task_state_[i], i);
      cancelable_id_[i] = task->id();
      V8::GetCurrentPlatform()->CallOnWorkerThread(std::move(task));
    }
  }
  DCHECK_EQ(total_task_count_, pending_task_count_);
}

void ConcurrentMarking::RescheduleTasksIfNeeded() {
  DCHECK(FLAG_parallel_marking || FLAG_concurrent_marking);
  if (heap_->IsTearingDown()) return;
  {
    base::MutexGuard guard(&pending_lock_);
    // The total task count is initialized in ScheduleTasks from
    // NumberOfWorkerThreads of the platform.
    if (total_task_count_ > 0 && pending_task_count_ == total_task_count_) {
      return;
    }
  }
  if (!shared_->IsGlobalPoolEmpty() ||
      !weak_objects_->current_ephemerons.IsGlobalPoolEmpty() ||
      !weak_objects_->discovered_ephemerons.IsGlobalPoolEmpty()) {
    ScheduleTasks();
  }
}

bool ConcurrentMarking::Stop(StopRequest stop_request) {
  DCHECK(FLAG_parallel_marking || FLAG_concurrent_marking);
  base::MutexGuard guard(&pending_lock_);

  if (pending_task_count_ == 0) return false;

  if (stop_request != StopRequest::COMPLETE_TASKS_FOR_TESTING) {
    CancelableTaskManager* task_manager =
        heap_->isolate()->cancelable_task_manager();
    for (int i = 1; i <= total_task_count_; i++) {
      if (is_pending_[i]) {
        if (task_manager->TryAbort(cancelable_id_[i]) ==
            TryAbortResult::kTaskAborted) {
          is_pending_[i] = false;
          --pending_task_count_;
        } else if (stop_request == StopRequest::PREEMPT_TASKS) {
          task_state_[i].preemption_request = true;
        }
      }
    }
  }
  while (pending_task_count_ > 0) {
    pending_condition_.Wait(&pending_lock_);
  }
  for (int i = 1; i <= total_task_count_; i++) {
    DCHECK(!is_pending_[i]);
  }
  return true;
}

bool ConcurrentMarking::IsStopped() {
  if (!FLAG_concurrent_marking) return true;

  base::MutexGuard guard(&pending_lock_);
  return pending_task_count_ == 0;
}

void ConcurrentMarking::FlushMemoryChunkData(
    MajorNonAtomicMarkingState* marking_state) {
  DCHECK_EQ(pending_task_count_, 0);
  for (int i = 1; i <= total_task_count_; i++) {
    MemoryChunkDataMap& memory_chunk_data = task_state_[i].memory_chunk_data;
    for (auto& pair : memory_chunk_data) {
      // ClearLiveness sets the live bytes to zero.
      // Pages with zero live bytes might be already unmapped.
      MemoryChunk* memory_chunk = pair.first;
      MemoryChunkData& data = pair.second;
      if (data.live_bytes) {
        marking_state->IncrementLiveBytes(memory_chunk, data.live_bytes);
      }
      if (data.typed_slots) {
        RememberedSet<OLD_TO_OLD>::MergeTyped(memory_chunk,
                                              std::move(data.typed_slots));
      }
    }
    memory_chunk_data.clear();
    task_state_[i].marked_bytes = 0;
  }
  total_marked_bytes_ = 0;
}

void ConcurrentMarking::ClearMemoryChunkData(MemoryChunk* chunk) {
  for (int i = 1; i <= total_task_count_; i++) {
    auto it = task_state_[i].memory_chunk_data.find(chunk);
    if (it != task_state_[i].memory_chunk_data.end()) {
      it->second.live_bytes = 0;
      it->second.typed_slots.reset();
    }
  }
}

size_t ConcurrentMarking::TotalMarkedBytes() {
  size_t result = 0;
  for (int i = 1; i <= total_task_count_; i++) {
    result +=
        base::AsAtomicWord::Relaxed_Load<size_t>(&task_state_[i].marked_bytes);
  }
  result += total_marked_bytes_;
  return result;
}

ConcurrentMarking::PauseScope::PauseScope(ConcurrentMarking* concurrent_marking)
    : concurrent_marking_(concurrent_marking),
      resume_on_exit_(FLAG_concurrent_marking &&
                      concurrent_marking_->Stop(
                          ConcurrentMarking::StopRequest::PREEMPT_TASKS)) {
  DCHECK_IMPLIES(resume_on_exit_, FLAG_concurrent_marking);
}

ConcurrentMarking::PauseScope::~PauseScope() {
  if (resume_on_exit_) concurrent_marking_->RescheduleTasksIfNeeded();
}

}  // namespace internal
}  // namespace v8