AssemblyHelpers.cpp

/*
 * Copyright (C) 2011-2025 Apple Inc. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "config.h"
#include "AssemblyHelpers.h"

WTF_ALLOW_UNSAFE_BUFFER_USAGE_BEGIN

#if ENABLE(JIT)

#include "AccessCase.h"
#include "AssemblyHelpersSpoolers.h"
#include "BaselineJITCode.h"
#include "ConcurrentButterflyOperations.h"
#include "JITOperations.h"
#include "JSArrayBufferView.h"
#include "JSCConfig.h"
#include "JSCJSValueInlines.h"
#include "JSDataView.h"
#include "LinkBuffer.h"
#include "MaxFrameExtentForSlowPathCall.h"
#include "MegamorphicCache.h"
#include "SuperSampler.h"
#include "ThunkGenerators.h"
#include "UnlinkedCodeBlock.h"
#include <atomic>
#include <mutex>

#if OS(DARWIN) && ENABLE(FAST_TLS_JIT)
#include <wtf/FastTLS.h>
#endif

#if ENABLE(WEBASSEMBLY)
#include "JSWebAssemblyInstance.h"
#include "WasmContext.h"
#include "WasmMemoryInformation.h"
#endif

namespace JSC {

namespace AssemblyHelpersInternal {
constexpr bool dumpVerbose = false;

// SPEC-jit section 5.5 (Task 8): frozen butterfly-tag encoding
// (SPEC-objectmodel section 2). Mirrored locally to keep this file
// independent of the object-model workstream's header; the values are FROZEN.
[[maybe_unused]] constexpr uint64_t butterflyTagPointerMask = 0x0000ffffffffffffULL;
[[maybe_unused]] constexpr uint64_t butterflyTagFloor = 1ULL << 48; // any word >= this carries tag bits

// Flag-on guard for the LEGACY (no slow-path list) property accessors: these
// remain reachable only from emissions whose butterflies are provably
// tag-free (gated transition/delete/megamorphic forms - see the Task 8 site
// inventory in docs/threads/INTEGRATE-jit.md) and from DFG/FTL emitters that
// Tasks 9/10 convert. A tagged or segmented word traps here instead of being
// misread as a pointer (I14 enforcement-by-construction; flag-off this emits
// nothing). Flag-on callers with a slow path must use
// CCallHelpers::loadProperty/storeProperty(..., slowCases) or the
// loadButterflyForRead/ForWrite choke points.
static void emitLegacyButterflyTagTrap(AssemblyHelpers& jit, GPRReg butterflyGPR)
{
#if USE(JSVALUE64)
    if (Options::useJSThreads()) [[unlikely]] {
        auto untagged = jit.branch64(AssemblyHelpers::Below, butterflyGPR, AssemblyHelpers::TrustedImm64(static_cast<int64_t>(butterflyTagFloor)));
        jit.breakpoint();
        untagged.link(&jit);
    }
#else
    UNUSED_PARAM(jit);
    UNUSED_PARAM(butterflyGPR);
#endif
}

// SPEC-jit section 5.5 Task 8 (I14(a)) + r48 (FUZZ.md / SCALEBENCH §48):
// load the ArrayBuffer* from a Wasteful typed-array view's butterfly,
// flag-on segment-aware. The previous "typed-array wasteful-mode butterflies
// are never segmented" claim is FALSE — a foreign-thread named-property add
// that escapes E4 (TTL fired / foreign TID) and grows outOfLineCapacity
// segments via trySegmentedTransition (the §44 StayFlatShared gate requires
// !hasIndexingHeader, which a Wasteful view HAS). Under that conversion the
// IndexingHeader (and its u.typedArray.buffer) lives at indexed fragment 0
// slot 0 (§4.1; the I8 alias equation pins it to the flat-era B-8 location,
// so the buffer pointer survives the conversion verbatim). resultGPR ==
// arrayBuffer on exit; scratchGPR is clobbered flag-on (callers re-load
// m_mode afterwards). Flag-off codegen unchanged: one masked loadPtr at
// [butterfly + offsetOfArrayBuffer].
static void emitLoadTypedArrayArrayBuffer(AssemblyHelpers& jit, GPRReg baseGPR, GPRReg resultGPR, [[maybe_unused]] GPRReg scratchGPR)
{
    jit.loadPtr(AssemblyHelpers::Address(baseGPR, JSObject::butterflyOffset()), resultGPR);
#if USE(JSVALUE64)
    if (Options::useJSThreads()) [[unlikely]] {
        ASSERT(scratchGPR != resultGPR && scratchGPR != baseGPR);
        // Segmented iff TID == notTTLTID (the all-ones TID, I3; payload != 0
        // is given by m_mode==Wasteful => the wastage install happened).
        constexpr uint64_t tidMask = 0x7fffULL << 48;
        static_assert(tidMask == JSC::butterflyTIDMask);
        jit.move(AssemblyHelpers::TrustedImm64(static_cast<int64_t>(tidMask)), scratchGPR);
        jit.and64(resultGPR, scratchGPR);
        auto notSegmented = jit.branch64(AssemblyHelpers::NotEqual, scratchGPR, AssemblyHelpers::TrustedImm64(static_cast<int64_t>(tidMask)));
        // Segmented: resultGPR = spine after mask; arrayBuffer is at
        // [indexedFragment(0) + 0] = [[spine + 32 + outOfLineFragmentCount*8] + 0].
        jit.and64(AssemblyHelpers::TrustedImm64(static_cast<int64_t>(butterflyTagPointerMask)), resultGPR);
        jit.load32(AssemblyHelpers::Address(resultGPR, 0), scratchGPR); // ButterflySpine::outOfLineFragmentCount
        static_assert(!OBJECT_OFFSETOF(ButterflySpine, outOfLineFragmentCount));
        static_assert(sizeof(ButterflySpine) == 32 && sizeof(void*) == 8);
        jit.loadPtr(AssemblyHelpers::BaseIndex(resultGPR, scratchGPR, AssemblyHelpers::TimesEight, sizeof(ButterflySpine)), resultGPR); // indexedFragment(0)
        static_assert(!OBJECT_OFFSETOF(ButterflyFragment, slots));
        static_assert(!IndexingHeader::offsetOfArrayBuffer()); // u.typedArray.buffer at offset 0 of the header slot.
        jit.loadPtr(AssemblyHelpers::Address(resultGPR, 0), resultGPR);
        auto done = jit.jump();
        notSegmented.link(&jit);
        jit.and64(AssemblyHelpers::TrustedImm64(static_cast<int64_t>(butterflyTagPointerMask)), resultGPR);
        jit.loadPtr(AssemblyHelpers::Address(resultGPR, Butterfly::offsetOfArrayBuffer()), resultGPR);
        done.link(&jit);
        return;
    }
#endif
    jit.loadPtr(AssemblyHelpers::Address(resultGPR, Butterfly::offsetOfArrayBuffer()), resultGPR);
}

}

// ===========================================================================
// UNGIL §A.1.1 (SPEC-ungil A.1.1; UNGIL-HANDOUT U-T3): loadVMLite — the
// one-load read of the current thread's VMLite* (the §A.1.3 GIL-off Group-3
// base; vmstate L4). Per-OS mechanics mirror SPEC-jit annex App. R5 exactly
// (the loadButterflyTIDTag precedent, jit/CCallHelpers.cpp /
// jit/ConcurrentButterflyOperations.cpp):
//
// - ELF (Linux glibc+musl, x86-64/arm64): one load off the thread register
//   at the initial-exec TLS offset of g_jscCurrentVMLite (the JIT/LLInt-
//   visible mirror of VMLite.cpp's `t_currentVMLite`, defined in
//   runtime/VM.cpp; VMLite::setCurrent keeps it coherent — the same
//   post-TLS-write discipline as the App. R5 CS3 TID-tag hook). IE-model
//   offsets are link-time thread-invariant; we still latch the first
//   computed value and RELEASE_ASSERT constancy at every emission (DFG/FTL
//   emit from multiple compiler threads, so this re-checks the App. R5
//   constancy property for free).
// - Darwin: Mach-O TLV has no constant offset, so the mirror lives in a
//   pthread TSD slot whose key is published through the M4a-style JSCConfig
//   slot (g_jscConfig.vmLiteTLSKey, beside butterflyTIDTagTLSKey;
//   JSC_CONFIG_HAS_VMLITE_TLS_KEY). pthread_key_create + per-thread
//   pthread_setspecific are the VMLite::setCurrent side's duty, exactly as
//   for the TID tag.
// - Other (Windows): unsupported flag-on per App. R5 — no new story owed.
//
// REMATERIALIZATION is the correctness carrier (§A.1.2): every site needing
// the lite may simply re-emit this load; prologue temps and the
// VMEntryRecord::m_vmLite slot are optimizations only.
//
// Flag-off / GIL-on identity: nothing calls this emitter except gilOff-mode
// compilations (§A.1.3 COMPILED-FOR-VM-mode rule, U-T4a/U-T4b) — flag-off
// golden disasm is unchanged by this definition existing.
// ===========================================================================

#if OS(LINUX) && (CPU(X86_64) || CPU(ARM64))

// Defined in runtime/VM.cpp (initial-exec model there too); declared with the
// same language linkage + TLS model so the address-of below resolves to the
// thread-invariant TPOFF the baked immediate relies on.
extern "C" __attribute__((tls_model("initial-exec"))) thread_local VMLite* g_jscCurrentVMLite;

namespace {

ALWAYS_INLINE uintptr_t currentThreadPointerForVMLiteTLS()
{
    uintptr_t threadPointer;
#if CPU(X86_64)
    // x86-64 ELF TLS ABI (glibc and musl): the TCB self-pointer lives at
    // %fs:0, so this single load yields the thread pointer the JIT's
    // fs-prefixed loads are relative to.
    asm volatile("movq %%fs:0, %0" : "=r"(threadPointer));
#elif CPU(ARM64)
    asm volatile("mrs %0, tpidr_el0" : "=r"(threadPointer));
#endif
    return threadPointer;
}

intptr_t currentVMLiteELFTLSOffset()
{
    intptr_t offset = static_cast<intptr_t>(
        reinterpret_cast<uintptr_t>(&g_jscCurrentVMLite) - currentThreadPointerForVMLiteTLS());
    // Both per-arch loadFromELFTLS64 emitters encode the offset as a
    // (sign-extended) disp32.
    RELEASE_ASSERT(offset == static_cast<intptr_t>(static_cast<int32_t>(offset)));
    static std::once_flag onceFlag;
    static intptr_t latchedOffset;
    std::call_once(onceFlag, [&] {
        latchedOffset = offset;
    });
    // App. R5 constancy property, re-verified per emission (and hence across
    // every compiler thread that ever emits this load).
    RELEASE_ASSERT(latchedOffset == offset);
    return offset;
}

} // anonymous namespace

#endif // OS(LINUX) && (CPU(X86_64) || CPU(ARM64))

// Free-function form: self-contained in this TU (compiles in any tree
// slice). The member spelling `jit.loadVMLite(reg)` — the surface U-T4a/b
// emission calls, mirroring CCallHelpers::loadButterflyTIDTag — forwards
// here once AssemblyHelpers.h (owned by the emission task per the U-T3/U-T4
// file split) declares it and defines JSC_ASSEMBLYHELPERS_HAS_LOAD_VMLITE
// beside the declaration (the JSC_CONFIG_HAS_BUTTERFLY_TID_TAG_TLS_KEY
// inversion pattern).
void loadVMLite(AssemblyHelpers&, GPRReg); // self-declaration (no header owns this form yet)
void loadVMLite(AssemblyHelpers& jit, GPRReg destGPR)
{
#if OS(LINUX) && (CPU(X86_64) || CPU(ARM64))
    // ARM64 note (mirrors loadFromTLS64): the offset is materialized through
    // destGPR itself for encodable offsets; load64 falls back to the data
    // temp for unencodable ones, so destGPR must not be the data temp.
    jit.loadFromELFTLS64(currentVMLiteELFTLSOffset(), destGPR);
#elif OS(DARWIN) && ENABLE(FAST_TLS_JIT)
#if defined(JSC_CONFIG_HAS_VMLITE_TLS_KEY)
    // TSD slots are uniform, so direct-offset reads via fastTLSOffsetForKey
    // are valid for dynamically created keys (App. R5 Darwin mechanics). The
    // key is created (and the per-thread copy maintained) by the
    // VMLite::setCurrent side before any gilOff-mode compilation can run —
    // U0c fixes the mode pre-codegen, and a gilOff compilation implies a
    // registered, installed lite existed.
    uint32_t key = g_jscConfig.vmLiteTLSKey;
    RELEASE_ASSERT(key);
    jit.loadFromTLS64(fastTLSOffsetForKey(key), destGPR);
#else
    // OPEN OBLIGATION (App. R5 Darwin mechanics, normative; escalated at
    // the U-T3 amendment): the JSCConfig vmLiteTLSKey slot (+
    // JSC_CONFIG_HAS_VMLITE_TLS_KEY beside it), its pthread_key_create at
    // the P5-init point that creates the TID-tag key, and the
    // VMLite::setCurrent-side per-thread pthread_setspecific are NOT YET
    // LANDED — runtime/JSCConfig.h and runtime/VMLite.cpp are outside this
    // slice's writable file set, and no IU obligation row yet names an
    // owner. They MUST land (with an IU row) before any task emits this
    // path on Darwin. Until then gilOff-mode compilation (the only caller)
    // fail-stops here rather than emitting a wrong load.
    UNUSED_PARAM(jit);
    UNUSED_PARAM(destGPR);
    RELEASE_ASSERT_NOT_REACHED();
#endif
#else
    // App. R5: no JIT-visible TLS mechanism on this platform; useJSThreads
    // (and a fortiori gilOffProcess) is unsupported here, so emission must
    // never get this far.
    UNUSED_PARAM(jit);
    UNUSED_PARAM(destGPR);
    RELEASE_ASSERT_NOT_REACHED();
#endif
}

#if defined(JSC_ASSEMBLYHELPERS_HAS_LOAD_VMLITE)
// Member surface (declared in AssemblyHelpers.h by the emission-task slice;
// the macro is defined beside the declaration). §A.1.2: callers may
// rematerialize freely — this is one TLS-relative load, no side effects.
void AssemblyHelpers::loadVMLite(GPRReg destGPR)
{
    JSC::loadVMLite(*this, destGPR);
}
#endif

// SPEC-jit App. R5: hoisted from CCallHelpers so emitTagInstalledButterflyWithTID
// (and the emitAllocateJSObject* templates) can call it. Mechanics are the
// loadVMLite precedent above (ELF IE-TLS / Darwin TSD, offset baked at
// emission via butterflyTIDTagELFTLSOffset / butterflyTIDTagTLSKey from
// ConcurrentButterflyOperations.h). CCallHelpers / SpeculativeJIT inherit it
// unchanged, so every existing call site (loadButterflyForWrite, the DFG/FTL
// §5.5 predicate emitters, the FTL loadButterflyTIDTag patchpoint generator)
// keeps compiling.
void AssemblyHelpers::loadButterflyTIDTag(GPRReg destGPR)
{
#if OS(LINUX) && (CPU(X86_64) || CPU(ARM64))
    loadFromELFTLS64(butterflyTIDTagELFTLSOffset(), destGPR);
#elif OS(DARWIN) && ENABLE(FAST_TLS_JIT)
    loadFromTLS64(fastTLSOffsetForKey(butterflyTIDTagTLSKey()), destGPR);
#else
    // D8/App. R5: no JIT-visible TLS mechanism; useJSThreads is unsupported
    // on this platform, so emission must never get here.
    UNUSED_PARAM(destGPR);
    RELEASE_ASSERT_NOT_REACHED();
#endif
}

// Task-8 (SPEC-objectmodel §2.1): see the declaration's comment. Kept
// out-of-line so AssemblyHelpers.h doesn't need ConcurrentButterflyOperations.h.
void AssemblyHelpers::emitTagInstalledButterflyWithTID(GPRReg resultGPR, GPRReg storageGPR, GPRReg scratchGPR)
{
#if USE(JSVALUE64)
    ASSERT(scratchGPR != resultGPR);
    ASSERT(scratchGPR != storageGPR);
    loadButterflyTIDTag(scratchGPR);
    // scratchGPR := (currentButterflyTID() << 48) | storageGPR — i.e.
    // encodeButterfly(storageGPR, currentButterflyTID(), false). storageGPR is
    // left untagged for the caller's post-install header/element writes.
    or64(storageGPR, scratchGPR);
    storePtr(scratchGPR, Address(resultGPR, JSObject::butterflyOffset()));
#else
    UNUSED_PARAM(resultGPR);
    UNUSED_PARAM(storageGPR);
    UNUSED_PARAM(scratchGPR);
    RELEASE_ASSERT_NOT_REACHED(); // flag-on requires 64-bit (D8)
#endif
}

AssemblyHelpers::Jump AssemblyHelpers::branchIfFastTypedArray(GPRReg baseGPR)
{
    return branch8(
        Equal,
        Address(baseGPR, JSArrayBufferView::offsetOfMode()),
        TrustedImm32(FastTypedArray));
}

AssemblyHelpers::Jump AssemblyHelpers::branchIfNotFastTypedArray(GPRReg baseGPR)
{
    return branch8(
        NotEqual,
        Address(baseGPR, JSArrayBufferView::offsetOfMode()),
        TrustedImm32(FastTypedArray));
}

void AssemblyHelpers::incrementSuperSamplerCount()
{
    add32(TrustedImm32(1), AbsoluteAddress(std::bit_cast<const void*>(&g_superSamplerCount)));
}

void AssemblyHelpers::decrementSuperSamplerCount()
{
    sub32(TrustedImm32(1), AbsoluteAddress(std::bit_cast<const void*>(&g_superSamplerCount)));
}

void AssemblyHelpers::purifyNaN(FPRReg inputFPR, FPRReg resultFPR)
{
    ASSERT(inputFPR != fpTempRegister);
#if CPU(ADDRESS64)
    move64ToDouble(TrustedImm64(std::bit_cast<uint64_t>(PNaN)), fpTempRegister);
    moveDoubleConditionallyDouble(DoubleEqualAndOrdered, inputFPR, inputFPR, inputFPR, fpTempRegister, resultFPR);
#else
    moveDouble(inputFPR, resultFPR);
    auto notNaN = branchIfNotNaN(resultFPR);
    move64ToDouble(TrustedImm64(std::bit_cast<uint64_t>(PNaN)), resultFPR);
    notNaN.link(this);
#endif
}

#if ENABLE(SAMPLING_FLAGS)
void AssemblyHelpers::setSamplingFlag(int32_t flag)
{
    ASSERT(flag >= 1);
    ASSERT(flag <= 32);
    or32(TrustedImm32(1u << (flag - 1)), AbsoluteAddress(SamplingFlags::addressOfFlags()));
}

void AssemblyHelpers::clearSamplingFlag(int32_t flag)
{
    ASSERT(flag >= 1);
    ASSERT(flag <= 32);
    and32(TrustedImm32(~(1u << (flag - 1))), AbsoluteAddress(SamplingFlags::addressOfFlags()));
}
#endif

#if ASSERT_ENABLED
#if USE(JSVALUE64)
void AssemblyHelpers::jitAssertIsInt32(GPRReg gpr)
{
    if (!Options::useJITAsserts())
        return;
#if CPU(X86_64) || CPU(ARM64)
    JIT_COMMENT(*this, "ASSERT is unboxed int32");
    Jump checkInt32 = branch64(BelowOrEqual, gpr, TrustedImm64(static_cast<uintptr_t>(0xFFFFFFFFu)));
    abortWithReason(AHIsNotInt32);
    checkInt32.link(this);
#else
    UNUSED_PARAM(gpr);
#endif
}

void AssemblyHelpers::jitAssertIsJSInt32(GPRReg gpr)
{
    if (!Options::useJITAsserts())
        return;
    JIT_COMMENT(*this, "ASSERT is JS boxed int32");
    Jump checkJSInt32 = branch64(AboveOrEqual, gpr, GPRInfo::numberTagRegister);
    abortWithReason(AHIsNotJSInt32);
    checkJSInt32.link(this);
}

void AssemblyHelpers::jitAssertIsJSNumber(GPRReg gpr)
{
    if (!Options::useJITAsserts())
        return;
    JIT_COMMENT(*this, "ASSERT is JS boxed number");
    Jump checkJSNumber = branchTest64(MacroAssembler::NonZero, gpr, GPRInfo::numberTagRegister);
    abortWithReason(AHIsNotJSNumber);
    checkJSNumber.link(this);
}

void AssemblyHelpers::jitAssertIsJSDouble(GPRReg gpr)
{
    if (!Options::useJITAsserts())
        return;
    JIT_COMMENT(*this, "ASSERT is JS boxed double (non-int32 number)");
    Jump checkJSInt32 = branch64(AboveOrEqual, gpr, GPRInfo::numberTagRegister);
    Jump checkJSNumber = branchTest64(MacroAssembler::NonZero, gpr, GPRInfo::numberTagRegister);
    checkJSInt32.link(this);
    abortWithReason(AHIsNotJSDouble);
    checkJSNumber.link(this);
}

void AssemblyHelpers::jitAssertIsCell(GPRReg gpr)
{
    if (!Options::useJITAsserts())
        return;
    JIT_COMMENT(*this, "ASSERT is JSCell");
    Jump checkCell = branchTest64(MacroAssembler::Zero, gpr, GPRInfo::notCellMaskRegister);
    abortWithReason(AHIsNotCell);
    checkCell.link(this);
}

void AssemblyHelpers::jitAssertTagsInPlace()
{
    if (!Options::useJITAsserts())
        return;
    Jump ok = branch64(Equal, GPRInfo::numberTagRegister, TrustedImm64(JSValue::NumberTag));
    abortWithReason(AHNumberTagNotInPlace);
    breakpoint();
    ok.link(this);

    ok = branch64(Equal, GPRInfo::notCellMaskRegister, TrustedImm64(JSValue::NotCellMask));
    abortWithReason(AHNotCellMaskNotInPlace);
    ok.link(this);
}
#elif USE(JSVALUE32_64)
void AssemblyHelpers::jitAssertIsInt32(GPRReg gpr)
{
    if (!Options::useJITAsserts())
        return;
    UNUSED_PARAM(gpr);
}

void AssemblyHelpers::jitAssertIsJSInt32(GPRReg gpr)
{
    if (!Options::useJITAsserts())
        return;
    Jump checkJSInt32 = branch32(Equal, gpr, TrustedImm32(JSValue::Int32Tag));
    abortWithReason(AHIsNotJSInt32);
    checkJSInt32.link(this);
}

void AssemblyHelpers::jitAssertIsJSNumber(GPRReg gpr)
{
    if (!Options::useJITAsserts())
        return;
    Jump checkJSInt32 = branch32(Equal, gpr, TrustedImm32(JSValue::Int32Tag));
    Jump checkJSDouble = branch32(Below, gpr, TrustedImm32(JSValue::LowestTag));
    abortWithReason(AHIsNotJSNumber);
    checkJSInt32.link(this);
    checkJSDouble.link(this);
}

void AssemblyHelpers::jitAssertIsJSDouble(GPRReg gpr)
{
    if (!Options::useJITAsserts())
        return;
    Jump checkJSDouble = branch32(Below, gpr, TrustedImm32(JSValue::LowestTag));
    abortWithReason(AHIsNotJSDouble);
    checkJSDouble.link(this);
}

void AssemblyHelpers::jitAssertIsCell(GPRReg gpr)
{
    if (!Options::useJITAsserts())
        return;
    Jump checkCell = branchIfCell(gpr);
    abortWithReason(AHIsNotCell);
    checkCell.link(this);
}

void AssemblyHelpers::jitAssertTagsInPlace()
{
    if (!Options::useJITAsserts())
        return;
}
#endif // USE(JSVALUE32_64)

void AssemblyHelpers::jitAssertHasValidCallFrame()
{
    if (!Options::useJITAsserts())
        return;
    Jump checkCFR = branchTestPtr(Zero, GPRInfo::callFrameRegister, TrustedImm32(7));
    abortWithReason(AHCallFrameMisaligned);
    checkCFR.link(this);
}

void AssemblyHelpers::jitAssertIsNull(GPRReg gpr)
{
    if (!Options::useJITAsserts())
        return;
    Jump checkNull = branchTestPtr(Zero, gpr);
    abortWithReason(AHIsNotNull);
    checkNull.link(this);
}

void AssemblyHelpers::jitAssertArgumentCountSane()
{
    if (!Options::useJITAsserts())
        return;
    Jump ok = branch32(Below, payloadFor(CallFrameSlot::argumentCountIncludingThis), TrustedImm32(10000000));
    abortWithReason(AHInsaneArgumentCount);
    ok.link(this);
}

void AssemblyHelpers::jitAssertCodeBlockOnCallFrameWithType(GPRReg scratchGPR, JITType type)
{
    if (!Options::useJITAsserts())
        return;
    JIT_COMMENT(*this, "jitAssertCodeBlockOnCallFrameWithType | ", scratchGPR, " = callFrame->codeBlock->jitCode->jitType == ", type);
    emitGetFromCallFrameHeaderPtr(CallFrameSlot::codeBlock, scratchGPR);
    loadPtr(Address(scratchGPR, CodeBlock::jitCodeOffset()), scratchGPR);
    load8(Address(scratchGPR, JITCode::offsetOfJITType()), scratchGPR);
    Jump ok = branch32(Equal, scratchGPR, TrustedImm32(static_cast<unsigned>(type)));
    abortWithReason(AHInvalidCodeBlock);
    ok.link(this);
}

void AssemblyHelpers::jitAssertCodeBlockMatchesCurrentCalleeCodeBlockOnCallFrame(GPRReg scratchGPR, GPRReg scratchGPR2, UnlinkedCodeBlock& block)
{
    if (!Options::useJITAsserts())
        return;
    // UNGIL FIX-2 (gilOff mode-split): with N mutators,
    // executable->codeBlockFor(kind) is a moving publish slot — tier-up
    // republish (ScriptExecutable::installCode, IT-8) and post-jettison
    // reinstall legitimately advance it while activations dispatched through
    // a coherent-but-stale CallLinkRecord are still entering or executing the
    // OLD CodeBlock (deferred invalidation, SPEC-jit I21: resumed mutators
    // keep executing replaced code until their next invalidation point).
    // "callFrame->codeBlock == the executable's CURRENTLY published
    // codeBlock" is therefore not an invariant gilOff, and the load emitted
    // below is unordered against the installer's fences anyway. Benign by
    // design => do not emit the check. Emission-time gate on the sticky
    // Config-page byte (same gate as ~CallLinkInfo): the gate is evaluated
    // while GENERATING code, so flag-off/GIL-on emitted code is
    // byte-identical and the V5b flag-off bench rule is untouched.
    // Accepted loss (review record): this was the only check that could catch
    // a same-tier WRONG-FUNCTION CodeBlock in the frame slot; the retained
    // canaries at jitAssertCodeBlockOnCallFrameWithType and
    // jitAssertCodeBlockOnCallFrameIsOptimizingJIT only see tier mismatches.
    if (g_jscConfig.gilOffProcess) [[unlikely]]
        return;
    if (block.codeType() != FunctionCode)
        return;
    auto kind = block.isConstructor() ? CodeSpecializationKind::CodeForConstruct : CodeSpecializationKind::CodeForCall;
    JIT_COMMENT(*this, "jitAssertCodeBlockMatchesCurrentCalleeCodeBlockOnCallFrame with code block type: ", kind, " | ", scratchGPR, " = callFrame->callee->executableOrRareData");

    emitGetFromCallFrameHeaderPtr(CallFrameSlot::callee, scratchGPR);
    loadPtr(Address(scratchGPR, JSFunction::offsetOfExecutableOrRareData()), scratchGPR);
    auto hasExecutable = branchTestPtr(Zero, scratchGPR, TrustedImm32(JSFunction::rareDataTag));
    loadPtr(Address(scratchGPR, FunctionRareData::offsetOfExecutable() - JSFunction::rareDataTag), scratchGPR);
    hasExecutable.link(this);
    JIT_COMMENT(*this, scratchGPR, " = (", scratchGPR, ": Executable)->codeBlock");
    loadPtr(Address(scratchGPR, FunctionExecutable::offsetOfCodeBlockFor(kind)), scratchGPR);

    JIT_COMMENT(*this, scratchGPR2, " = callFrame->codeBlock");
    emitGetFromCallFrameHeaderPtr(CallFrameSlot::codeBlock, scratchGPR2);
    Jump ok = branch32(Equal, scratchGPR, scratchGPR2);
    abortWithReason(AHInvalidCodeBlock);
    ok.link(this);
}

void AssemblyHelpers::jitAssertCodeBlockOnCallFrameIsOptimizingJIT(GPRReg scratchGPR)
{
    if (!Options::useJITAsserts())
        return;
    emitGetFromCallFrameHeaderPtr(CallFrameSlot::codeBlock, scratchGPR);
    loadPtr(Address(scratchGPR, CodeBlock::jitCodeOffset()), scratchGPR);
    load8(Address(scratchGPR, JITCode::offsetOfJITType()), scratchGPR);
    JumpList ok;
    ok.append(branch32(Equal, scratchGPR, TrustedImm32(static_cast<unsigned>(JITType::DFGJIT))));
    ok.append(branch32(Equal, scratchGPR, TrustedImm32(static_cast<unsigned>(JITType::FTLJIT))));
    abortWithReason(AHInvalidCodeBlock);
    ok.link(this);
}

#endif // ASSERT_ENABLED

// UNGIL §A.1.3 (U-T4): see the declaration comment in AssemblyHelpers.h.
AssemblyHelpers::Address AssemblyHelpers::materializeGILOffExceptionSlot()
{
#if CPU(ARM64)
    // Cache-invalidating accessor for the same reason as prepareCallOperation:
    // loadVMLite writes the temp via mrs+ldr without updating
    // m_cachedMemoryTempRegister.
    GPRReg scratchGPR = getCachedMemoryTempRegisterIDAndInvalidate();
#elif CPU(X86_64)
    GPRReg scratchGPR = scratchRegister(); // r11, already clobbered by the GIL-on AbsoluteAddress form.
#else
    // SPEC-jit annex App. R5: no gilOff support here; loadVMLite fail-stops
    // at emission before this address is used.
    GPRReg scratchGPR = GPRInfo::nonArgGPR0;
#endif
    loadVMLite(scratchGPR);
    return Address(scratchGPR, static_cast<int32_t>(VMLite::offsetOfPrimitives() + VMLitePrimitives::offsetOf_m_exception()));
}

void AssemblyHelpers::loadException(VM& vm, GPRReg destGPR)
{
    if (vm.gilOff()) [[unlikely]] {
        loadVMLite(destGPR);
        loadPtr(Address(destGPR, static_cast<int32_t>(VMLite::offsetOfPrimitives() + VMLitePrimitives::offsetOf_m_exception())), destGPR);
    } else
        loadPtr(vm.addressOfException(), destGPR);
}

void AssemblyHelpers::jitReleaseAssertNoException(VM& vm)
{
    Jump noException;
#if USE(JSVALUE64)
    if (vm.gilOff()) [[unlikely]]
        noException = branchTestPtr(Zero, materializeGILOffExceptionSlot());
    else
        noException = branchTest64(Zero, AbsoluteAddress(vm.addressOfException()));
#elif USE(JSVALUE32_64)
    // GIL-off is unsupported on 32-bit platforms (jit App. R5); loadVMLite
    // fail-stops at emission, so the absolute form stays correct here.
    noException = branch32(Equal, AbsoluteAddress(vm.addressOfException()), TrustedImm32(0));
#endif
    abortWithReason(JITUncaughtExceptionAfterCall);
    noException.link(this);
}

void AssemblyHelpers::callExceptionFuzz(VM& vm, GPRReg exceptionReg)
{
    RELEASE_ASSERT(Options::useExceptionFuzz());

    EncodedJSValue* buffer = vm.exceptionFuzzingBuffer(sizeof(EncodedJSValue) * (GPRInfo::numberOfRegisters + FPRInfo::numberOfRegisters));

    for (unsigned i = 0; i < GPRInfo::numberOfRegisters; ++i) {
#if USE(JSVALUE64)
        store64(GPRInfo::toRegister(i), buffer + i);
#else
        store32(GPRInfo::toRegister(i), buffer + i);
#endif
    }
    for (unsigned i = 0; i < FPRInfo::numberOfRegisters; ++i) {
        move(TrustedImmPtr(buffer + GPRInfo::numberOfRegisters + i), GPRInfo::regT0);
        storeDouble(FPRInfo::toRegister(i), Address(GPRInfo::regT0));
    }

    // Set up one argument.
    move(TrustedImmPtr(&vm), GPRInfo::argumentGPR0);
    move(TrustedImmPtr(tagCFunction<OperationPtrTag>(operationExceptionFuzzWithCallFrame)), GPRInfo::nonPreservedNonReturnGPR);
    prepareCallOperation(vm);
    call(GPRInfo::nonPreservedNonReturnGPR, OperationPtrTag);

    for (unsigned i = 0; i < FPRInfo::numberOfRegisters; ++i) {
        move(TrustedImmPtr(buffer + GPRInfo::numberOfRegisters + i), GPRInfo::regT0);
        loadDouble(Address(GPRInfo::regT0), FPRInfo::toRegister(i));
    }
    for (unsigned i = 0; i < GPRInfo::numberOfRegisters; ++i) {
#if USE(JSVALUE64)
        load64(buffer + i, GPRInfo::toRegister(i));
#else
        load32(buffer + i, GPRInfo::toRegister(i));
#endif
    }

    if (exceptionReg != InvalidGPRReg)
        loadException(vm, exceptionReg);
}

AssemblyHelpers::Jump AssemblyHelpers::emitJumpIfException(VM& vm)
{
    return emitExceptionCheck(vm, NormalExceptionCheck);
}

AssemblyHelpers::Jump AssemblyHelpers::emitExceptionCheck(VM& vm, ExceptionCheckKind kind, ExceptionJumpWidth width, GPRReg exceptionReg)
{
    if (Options::useExceptionFuzz()) [[unlikely]]
        callExceptionFuzz(vm, exceptionReg);

    if (width == FarJumpWidth)
        kind = (kind == NormalExceptionCheck ? InvertedExceptionCheck : NormalExceptionCheck);

    Jump result;
    if (exceptionReg != InvalidGPRReg) {
#if ASSERT_ENABLED
        JIT_COMMENT(*this, "Exception validation");
        Jump ok;
        if (vm.gilOff()) [[unlikely]]
            ok = branchPtr(Equal, materializeGILOffExceptionSlot(), exceptionReg);
        else
            ok = branchPtr(Equal, AbsoluteAddress(vm.addressOfException()), exceptionReg);
        breakpoint();
        ok.link(this);
#endif
        JIT_COMMENT(*this, "Exception check from operation result register");
        result = branchTestPtr(kind == NormalExceptionCheck ? NonZero : Zero, exceptionReg);
    } else {
        JIT_COMMENT(*this, "Exception check from vm");
        if (vm.gilOff()) [[unlikely]]
            result = branchTestPtr(kind == NormalExceptionCheck ? NonZero : Zero, materializeGILOffExceptionSlot());
        else
            result = branchTestPtr(kind == NormalExceptionCheck ? NonZero : Zero, AbsoluteAddress(vm.addressOfException()));
    }

    if (width == NormalJumpWidth)
        return result;

    PatchableJump realJump = patchableJump();
    result.link(this);

    return realJump.m_jump;
}

AssemblyHelpers::Jump AssemblyHelpers::emitNonPatchableExceptionCheck(VM& vm, GPRReg exceptionReg)
{
    return emitExceptionCheck(vm, NormalExceptionCheck, NormalJumpWidth, exceptionReg);
}

void AssemblyHelpers::emitStoreStructureWithTypeInfo(AssemblyHelpers& jit, TrustedImmPtr structure, RegisterID dest)
{
    const Structure* structurePtr = reinterpret_cast<const Structure*>(structure.m_value);
#if USE(JSVALUE64)
    jit.store64(TrustedImm64(static_cast<uint64_t>(structurePtr->id().bits()) | (static_cast<uint64_t>(structurePtr->typeInfoBlob()) << 32)), MacroAssembler::Address(dest, JSCell::structureIDOffset()));
    if (ASSERT_ENABLED) {
        Jump correctStructure = jit.branch32(Equal, MacroAssembler::Address(dest, JSCell::structureIDOffset()), TrustedImm32(structurePtr->id().bits()));
        jit.abortWithReason(AHStructureIDIsValid);
        correctStructure.link(&jit);

        Jump correctIndexingType = jit.branch8(Equal, MacroAssembler::Address(dest, JSCell::indexingTypeAndMiscOffset()), TrustedImm32(structurePtr->indexingModeIncludingHistory()));
        jit.abortWithReason(AHIndexingTypeIsValid);
        correctIndexingType.link(&jit);

        Jump correctType = jit.branch8(Equal, MacroAssembler::Address(dest, JSCell::typeInfoTypeOffset()), TrustedImm32(structurePtr->typeInfo().type()));
        jit.abortWithReason(AHTypeInfoIsValid);
        correctType.link(&jit);

        Jump correctFlags = jit.branch8(Equal, MacroAssembler::Address(dest, JSCell::typeInfoFlagsOffset()), TrustedImm32(structurePtr->typeInfo().inlineTypeFlags()));
        jit.abortWithReason(AHTypeInfoInlineTypeFlagsAreValid);
        correctFlags.link(&jit);
    }
#else
    // Do a 32-bit wide store to initialize the cell's fields.
    jit.store32(TrustedImm32(structurePtr->typeInfoBlob()), MacroAssembler::Address(dest, JSCell::indexingTypeAndMiscOffset()));
    jit.storePtr(structure, MacroAssembler::Address(dest, JSCell::structureIDOffset()));
#endif
}

void AssemblyHelpers::loadProperty(GPRReg object, GPRReg offset, JSValueRegs result)
{
    ASSERT(noOverlap(offset, result));
    Jump isInline = branch32(LessThan, offset, TrustedImm32(firstOutOfLineOffset));

    loadPtr(Address(object, JSObject::butterflyOffset()), result.payloadGPR());
    AssemblyHelpersInternal::emitLegacyButterflyTagTrap(*this, result.payloadGPR()); // SPEC-jit section 5.5 Task 8
    neg32(offset);
    signExtend32ToPtr(offset, offset);
    Jump ready = jump();

    isInline.link(this);
    addPtr(
        TrustedImm32(
            static_cast<int32_t>(JSObject::offsetOfInlineStorage()) -
            (static_cast<int32_t>(firstOutOfLineOffset) - 2) * static_cast<int32_t>(sizeof(EncodedJSValue))),
        object, result.payloadGPR());

    ready.link(this);

    loadValue(
        BaseIndex(
            result.payloadGPR(), offset, TimesEight, (firstOutOfLineOffset - 2) * sizeof(EncodedJSValue)),
        result);
}

void AssemblyHelpers::storeProperty(JSValueRegs value, GPRReg object, GPRReg offset, GPRReg scratch)
{
    // Actually, object can be the same to scratch.
    ASSERT(noOverlap(offset, scratch));
    ASSERT(noOverlap(value, scratch));
    Jump isInline = branch32(LessThan, offset, TrustedImm32(firstOutOfLineOffset));

    loadPtr(Address(object, JSObject::butterflyOffset()), scratch);
    AssemblyHelpersInternal::emitLegacyButterflyTagTrap(*this, scratch); // SPEC-jit section 5.5 Task 8
    neg32(offset);
    signExtend32ToPtr(offset, offset);
    Jump ready = jump();

    isInline.link(this);
    addPtr(
        TrustedImm32(
            static_cast<int32_t>(JSObject::offsetOfInlineStorage()) -
            (static_cast<int32_t>(firstOutOfLineOffset) - 2) * static_cast<int32_t>(sizeof(EncodedJSValue))),
        object, scratch);

    ready.link(this);

    storeValue(value, BaseIndex(scratch, offset, TimesEight, (firstOutOfLineOffset - 2) * sizeof(EncodedJSValue)));
}

#if USE(JSVALUE64)
AssemblyHelpers::JumpList AssemblyHelpers::loadMegamorphicProperty(VM& vm, GPRReg baseGPR, GPRReg uidGPR, UniquedStringImpl* uid, GPRReg resultGPR, GPRReg scratch1GPR, GPRReg scratch2GPR, GPRReg scratch3GPR)
{
    if (Options::useJSThreads()) [[unlikely]] {
        // SPEC-jit section 5.5 (Task 8): the megamorphic fast path reads the
        // VM-global MegamorphicCache without synchronization and dereferences
        // the butterfly without the TID/SW predicate; flag-on every
        // megamorphic access defers to the generic operation (Task 8
        // inventory; revisit with vmstate's shared-cache story).
        JumpList slowCases;
        slowCases.append(jump());
        return slowCases;
    }

    // uidGPR can be InvalidGPRReg if uid is non-nullptr.

    if (!uid)
        ASSERT(uidGPR != InvalidGPRReg);

    JumpList primaryFail;
    JumpList slowCases;

    load32(Address(baseGPR, JSCell::structureIDOffset()), scratch1GPR);
#if CPU(ARM64)
    extractUnsignedBitfield32(scratch1GPR, TrustedImm32(MegamorphicCache::structureIDHashShift1), TrustedImm32(32 - MegamorphicCache::structureIDHashShift1), scratch2GPR);
    xorUnsignedRightShift32(scratch2GPR, scratch1GPR, TrustedImm32(MegamorphicCache::structureIDHashShift2), scratch3GPR);
#else
    urshift32(scratch1GPR, TrustedImm32(MegamorphicCache::structureIDHashShift1), scratch2GPR);
    urshift32(scratch1GPR, TrustedImm32(MegamorphicCache::structureIDHashShift2), scratch3GPR);
    xor32(scratch2GPR, scratch3GPR);
#endif
    if (uid)
        add32(TrustedImm32(uid->hash()), scratch3GPR);
    else {
        // Note that we don't test if the hash is zero here. AtomStringImpl's can't have a zero
        // hash, however, a SymbolImpl may. But, because this is a cache, we don't care. We only
        // ever load the result from the cache if the cache entry matches what we are querying for.
        // So we either get super lucky and use zero for the hash and somehow collide with the entity
        // we're looking for, or we realize we're comparing against another entity, and go to the
        // slow path anyways.
        load32(Address(uidGPR, UniquedStringImpl::flagsOffset()), scratch2GPR);
        urshift32(TrustedImm32(StringImpl::s_flagCount), scratch2GPR);
        add32(scratch2GPR, scratch3GPR);
    }

    and32(TrustedImm32(MegamorphicCache::loadCachePrimaryMask), scratch3GPR);
    if (hasOneBitSet(sizeof(MegamorphicCache::LoadEntry))) // is a power of 2
        lshift32(TrustedImm32(getLSBSet(sizeof(MegamorphicCache::LoadEntry))), scratch3GPR);
    else
        mul32(TrustedImm32(sizeof(MegamorphicCache::LoadEntry)), scratch3GPR, scratch3GPR);
    auto& cache = vm.ensureMegamorphicCache();
    move(TrustedImmPtr(&cache), scratch2GPR);
    static_assert(!MegamorphicCache::offsetOfLoadCachePrimaryEntries());
    addPtr(scratch2GPR, scratch3GPR);

    load16(Address(scratch2GPR, MegamorphicCache::offsetOfEpoch()), scratch2GPR);

    primaryFail.append(branch32(NotEqual, scratch1GPR, Address(scratch3GPR, MegamorphicCache::LoadEntry::offsetOfStructureID())));
    if (uid)
        primaryFail.append(branchPtr(NotEqual, Address(scratch3GPR, MegamorphicCache::LoadEntry::offsetOfUid()), TrustedImmPtr(uid)));
    else
        primaryFail.append(branchPtr(NotEqual, Address(scratch3GPR, MegamorphicCache::LoadEntry::offsetOfUid()), uidGPR));
    // We already hit StructureID and uid. And we get stale epoch for this entry.
    // Since all entries in the secondary cache has stale epoch for this StructureID and uid pair, we should just go to the slow case.
    slowCases.append(branch32WithMemory16(NotEqual, Address(scratch3GPR, MegamorphicCache::LoadEntry::offsetOfEpoch()), scratch2GPR));

    // Cache hit!
    Label cacheHit = label();
    loadPtr(Address(scratch3GPR, MegamorphicCache::LoadEntry::offsetOfHolder()), scratch2GPR);
    auto missed = branchTestPtr(Zero, scratch2GPR);
    moveConditionally64(Equal, scratch2GPR, TrustedImm32(std::bit_cast<uintptr_t>(JSCell::seenMultipleCalleeObjects())), baseGPR, scratch2GPR, scratch1GPR);
    load16(Address(scratch3GPR, MegamorphicCache::LoadEntry::offsetOfOffset()), scratch2GPR);
    loadProperty(scratch1GPR, scratch2GPR, JSValueRegs { resultGPR });
    auto done = jump();

    // Secondary cache lookup. Now,
    //   1. scratch1GPR holds StructureID.
    //   2. scratch2GPR holds global epoch.
    primaryFail.link(this);
    if (uid)
        add32(TrustedImm32(static_cast<uint32_t>(std::bit_cast<uintptr_t>(uid))), scratch1GPR, scratch3GPR);
    else
        add32(uidGPR, scratch1GPR, scratch3GPR);
    addUnsignedRightShift32(scratch3GPR, scratch3GPR, TrustedImm32(MegamorphicCache::structureIDHashShift3), scratch3GPR);
    and32(TrustedImm32(MegamorphicCache::loadCacheSecondaryMask), scratch3GPR);
    if constexpr (hasOneBitSet(sizeof(MegamorphicCache::LoadEntry))) // is a power of 2
        lshift32(TrustedImm32(getLSBSet(sizeof(MegamorphicCache::LoadEntry))), scratch3GPR);
    else
        mul32(TrustedImm32(sizeof(MegamorphicCache::LoadEntry)), scratch3GPR, scratch3GPR);
    addPtr(TrustedImmPtr(std::bit_cast<uint8_t*>(&cache) + MegamorphicCache::offsetOfLoadCacheSecondaryEntries()), scratch3GPR);

    slowCases.append(branch32(NotEqual, scratch1GPR, Address(scratch3GPR, MegamorphicCache::LoadEntry::offsetOfStructureID())));
    if (uid)
        slowCases.append(branchPtr(NotEqual, Address(scratch3GPR, MegamorphicCache::LoadEntry::offsetOfUid()), TrustedImmPtr(uid)));
    else
        slowCases.append(branchPtr(NotEqual, Address(scratch3GPR, MegamorphicCache::LoadEntry::offsetOfUid()), uidGPR));
    slowCases.append(branch32WithMemory16(NotEqual, Address(scratch3GPR, MegamorphicCache::LoadEntry::offsetOfEpoch()), scratch2GPR));
    jump().linkTo(cacheHit, this);

    missed.link(this);
    moveTrustedValue(jsUndefined(), JSValueRegs { resultGPR });

    done.link(this);

    return slowCases;
}

std::tuple<AssemblyHelpers::JumpList, AssemblyHelpers::JumpList> AssemblyHelpers::storeMegamorphicProperty(VM& vm, GPRReg baseGPR, GPRReg uidGPR, UniquedStringImpl* uid, GPRReg valueGPR, GPRReg scratch1GPR, GPRReg scratch2GPR, GPRReg scratch3GPR)
{
    if (Options::useJSThreads()) [[unlikely]] {
        // SPEC-jit section 5.5 (Task 8): see loadMegamorphicProperty above.
        JumpList slowCases;
        slowCases.append(jump());
        return { WTF::move(slowCases), JumpList() };
    }

    // uidGPR can be InvalidGPRReg if uid is non-nullptr.

    if (!uid)
        ASSERT(uidGPR != InvalidGPRReg);

    JumpList primaryFail;
    JumpList slowCases;
    JumpList reallocatingCases;

    load32(Address(baseGPR, JSCell::structureIDOffset()), scratch1GPR);
#if CPU(ARM64)
    extractUnsignedBitfield32(scratch1GPR, TrustedImm32(MegamorphicCache::structureIDHashShift1), TrustedImm32(32 - MegamorphicCache::structureIDHashShift1), scratch2GPR);
    xorUnsignedRightShift32(scratch2GPR, scratch1GPR, TrustedImm32(MegamorphicCache::structureIDHashShift4), scratch3GPR);
#else
    urshift32(scratch1GPR, TrustedImm32(MegamorphicCache::structureIDHashShift1), scratch2GPR);
    urshift32(scratch1GPR, TrustedImm32(MegamorphicCache::structureIDHashShift4), scratch3GPR);
    xor32(scratch2GPR, scratch3GPR);
#endif

    if (uid)
        add32(TrustedImm32(uid->hash()), scratch3GPR);
    else {
        // Note that we don't test if the hash is zero here. AtomStringImpl's can't have a zero
        // hash, however, a SymbolImpl may. But, because this is a cache, we don't care. We only
        // ever load the result from the cache if the cache entry matches what we are querying for.
        // So we either get super lucky and use zero for the hash and somehow collide with the entity
        // we're looking for, or we realize we're comparing against another entity, and go to the
        // slow path anyways.
        load32(Address(uidGPR, UniquedStringImpl::flagsOffset()), scratch2GPR);
        urshift32(TrustedImm32(StringImpl::s_flagCount), scratch2GPR);
        add32(scratch2GPR, scratch3GPR);
    }

    and32(TrustedImm32(MegamorphicCache::storeCachePrimaryMask), scratch3GPR);
    if (hasOneBitSet(sizeof(MegamorphicCache::StoreEntry))) // is a power of 2
        lshift32(TrustedImm32(getLSBSet(sizeof(MegamorphicCache::StoreEntry))), scratch3GPR);
    else
        mul32(TrustedImm32(sizeof(MegamorphicCache::StoreEntry)), scratch3GPR, scratch3GPR);
    auto& cache = vm.ensureMegamorphicCache();
    move(TrustedImmPtr(&cache), scratch2GPR);
    addPtr(scratch2GPR, scratch3GPR);
    addPtr(TrustedImmPtr(MegamorphicCache::offsetOfStoreCachePrimaryEntries()), scratch3GPR);

    load16(Address(scratch2GPR, MegamorphicCache::offsetOfEpoch()), scratch2GPR);

    primaryFail.append(branch32(NotEqual, scratch1GPR, Address(scratch3GPR, MegamorphicCache::StoreEntry::offsetOfOldStructureID())));
    if (uid)
        primaryFail.append(branchPtr(NotEqual, Address(scratch3GPR, MegamorphicCache::StoreEntry::offsetOfUid()), TrustedImmPtr(uid)));
    else
        primaryFail.append(branchPtr(NotEqual, Address(scratch3GPR, MegamorphicCache::StoreEntry::offsetOfUid()), uidGPR));
    // We already hit StructureID and uid. And we get stale epoch for this entry.
    // Since all entries in the secondary cache has stale epoch for this StructureID and uid pair, we should just go to the slow case.
    slowCases.append(branch32WithMemory16(NotEqual, Address(scratch3GPR, MegamorphicCache::StoreEntry::offsetOfEpoch()), scratch2GPR));

    // Cache hit!
    Label cacheHit = label();
    reallocatingCases.append(branchTest8(NonZero, Address(scratch3GPR, MegamorphicCache::StoreEntry::offsetOfReallocating())));
    load32(Address(scratch3GPR, MegamorphicCache::StoreEntry::offsetOfNewStructureID()), scratch2GPR);
    load16(Address(scratch3GPR, MegamorphicCache::StoreEntry::offsetOfOffset()), scratch3GPR);
    auto replaceCase = branch32(Equal, scratch2GPR, scratch1GPR);

    // We only support non-allocating transition. This means we do not need to nuke Structure* for transition here.
    store32(scratch2GPR, Address(baseGPR, JSCell::structureIDOffset()));

    replaceCase.link(this);
    storeProperty(JSValueRegs { valueGPR }, baseGPR, scratch3GPR, scratch1GPR);
    auto done = jump();

    // Secondary cache lookup
    primaryFail.link(this);
    if (uid)
        add32(TrustedImm32(static_cast<uint32_t>(std::bit_cast<uintptr_t>(uid))), scratch1GPR, scratch3GPR);
    else
        add32(uidGPR, scratch1GPR, scratch3GPR);
    addUnsignedRightShift32(scratch3GPR, scratch3GPR, TrustedImm32(MegamorphicCache::structureIDHashShift5), scratch3GPR);
    and32(TrustedImm32(MegamorphicCache::storeCacheSecondaryMask), scratch3GPR);
    if constexpr (hasOneBitSet(sizeof(MegamorphicCache::StoreEntry))) // is a power of 2
        lshift32(TrustedImm32(getLSBSet(sizeof(MegamorphicCache::StoreEntry))), scratch3GPR);
    else
        mul32(TrustedImm32(sizeof(MegamorphicCache::StoreEntry)), scratch3GPR, scratch3GPR);
    addPtr(TrustedImmPtr(std::bit_cast<uint8_t*>(&cache) + MegamorphicCache::offsetOfStoreCacheSecondaryEntries()), scratch3GPR);

    slowCases.append(branch32(NotEqual, scratch1GPR, Address(scratch3GPR, MegamorphicCache::StoreEntry::offsetOfOldStructureID())));
    if (uid)
        slowCases.append(branchPtr(NotEqual, Address(scratch3GPR, MegamorphicCache::StoreEntry::offsetOfUid()), TrustedImmPtr(uid)));
    else
        slowCases.append(branchPtr(NotEqual, Address(scratch3GPR, MegamorphicCache::StoreEntry::offsetOfUid()), uidGPR));
    slowCases.append(branch32WithMemory16(NotEqual, Address(scratch3GPR, MegamorphicCache::StoreEntry::offsetOfEpoch()), scratch2GPR));
    jump().linkTo(cacheHit, this);

    done.link(this);

    return std::tuple { slowCases, reallocatingCases };
}

AssemblyHelpers::JumpList AssemblyHelpers::hasMegamorphicProperty(VM& vm, GPRReg baseGPR, GPRReg uidGPR, UniquedStringImpl* uid, GPRReg resultGPR, GPRReg scratch1GPR, GPRReg scratch2GPR, GPRReg scratch3GPR)
{
    if (Options::useJSThreads()) [[unlikely]] {
        // SPEC-jit section 5.5 (Task 8): see loadMegamorphicProperty above.
        // This emitter was missed by the Task 8 sweep: it reads the shared
        // HasEntry words (incl. the RefPtr'd uid slot) with no
        // synchronization. With MegamorphicCache fills disabled flag-on
        // (MegamorphicCache.h) a hit is impossible anyway; bail like the
        // load/store emitters.
        JumpList slowCases;
        slowCases.append(jump());
        return slowCases;
    }

    // uidGPR can be InvalidGPRReg if uid is non-nullptr.

    if (!uid)
        ASSERT(uidGPR != InvalidGPRReg);

    JumpList primaryFail;
    JumpList slowCases;

    load32(Address(baseGPR, JSCell::structureIDOffset()), scratch1GPR);
#if CPU(ARM64)
    extractUnsignedBitfield32(scratch1GPR, TrustedImm32(MegamorphicCache::structureIDHashShift1), TrustedImm32(32 - MegamorphicCache::structureIDHashShift1), scratch2GPR);
    xorUnsignedRightShift32(scratch2GPR, scratch1GPR, TrustedImm32(MegamorphicCache::structureIDHashShift6), scratch3GPR);
#else
    urshift32(scratch1GPR, TrustedImm32(MegamorphicCache::structureIDHashShift1), scratch2GPR);
    urshift32(scratch1GPR, TrustedImm32(MegamorphicCache::structureIDHashShift6), scratch3GPR);
    xor32(scratch2GPR, scratch3GPR);
#endif
    if (uid)
        add32(TrustedImm32(uid->hash()), scratch3GPR);
    else {
        // Note that we don't test if the hash is zero here. AtomStringImpl's can't have a zero
        // hash, however, a SymbolImpl may. But, because this is a cache, we don't care. We only
        // ever load the result from the cache if the cache entry matches what we are querying for.
        // So we either get super lucky and use zero for the hash and somehow collide with the entity
        // we're looking for, or we realize we're comparing against another entity, and go to the
        // slow path anyways.
        load32(Address(uidGPR, UniquedStringImpl::flagsOffset()), scratch2GPR);
        urshift32(TrustedImm32(StringImpl::s_flagCount), scratch2GPR);
        add32(scratch2GPR, scratch3GPR);
    }

    and32(TrustedImm32(MegamorphicCache::hasCachePrimaryMask), scratch3GPR);
    if (hasOneBitSet(sizeof(MegamorphicCache::HasEntry))) // is a power of 2
        lshift32(TrustedImm32(getLSBSet(sizeof(MegamorphicCache::HasEntry))), scratch3GPR);
    else
        mul32(TrustedImm32(sizeof(MegamorphicCache::HasEntry)), scratch3GPR, scratch3GPR);
    auto& cache = vm.ensureMegamorphicCache();
    move(TrustedImmPtr(&cache), scratch2GPR);
    addPtr(scratch2GPR, scratch3GPR);
    addPtr(TrustedImmPtr(MegamorphicCache::offsetOfHasCachePrimaryEntries()), scratch3GPR);

    load16(Address(scratch2GPR, MegamorphicCache::offsetOfEpoch()), scratch2GPR);

    primaryFail.append(branch32(NotEqual, scratch1GPR, Address(scratch3GPR, MegamorphicCache::HasEntry::offsetOfStructureID())));
    if (uid)
        primaryFail.append(branchPtr(NotEqual, Address(scratch3GPR, MegamorphicCache::HasEntry::offsetOfUid()), TrustedImmPtr(uid)));
    else
        primaryFail.append(branchPtr(NotEqual, Address(scratch3GPR, MegamorphicCache::HasEntry::offsetOfUid()), uidGPR));
    // We already hit StructureID and uid. And we get stale epoch for this entry.
    // Since all entries in the secondary cache has stale epoch for this StructureID and uid pair, we should just go to the slow case.
    slowCases.append(branch32WithMemory16(NotEqual, Address(scratch3GPR, MegamorphicCache::HasEntry::offsetOfEpoch()), scratch2GPR));

    // Cache hit!
    Label cacheHit = label();
    load16(Address(scratch3GPR, MegamorphicCache::HasEntry::offsetOfResult()), scratch2GPR);
    boxBoolean(scratch2GPR, JSValueRegs { resultGPR });
    auto done = jump();

    // Secondary cache lookup. Now,
    //   1. scratch1GPR holds StructureID.
    //   2. scratch2GPR holds global epoch.
    primaryFail.link(this);
    if (uid)
        add32(TrustedImm32(static_cast<uint32_t>(std::bit_cast<uintptr_t>(uid))), scratch1GPR, scratch3GPR);
    else
        add32(uidGPR, scratch1GPR, scratch3GPR);
    addUnsignedRightShift32(scratch3GPR, scratch3GPR, TrustedImm32(MegamorphicCache::structureIDHashShift7), scratch3GPR);
    and32(TrustedImm32(MegamorphicCache::hasCacheSecondaryMask), scratch3GPR);
    if constexpr (hasOneBitSet(sizeof(MegamorphicCache::HasEntry))) // is a power of 2
        lshift32(TrustedImm32(getLSBSet(sizeof(MegamorphicCache::HasEntry))), scratch3GPR);
    else
        mul32(TrustedImm32(sizeof(MegamorphicCache::HasEntry)), scratch3GPR, scratch3GPR);
    addPtr(TrustedImmPtr(std::bit_cast<uint8_t*>(&cache) + MegamorphicCache::offsetOfHasCacheSecondaryEntries()), scratch3GPR);

    slowCases.append(branch32(NotEqual, scratch1GPR, Address(scratch3GPR, MegamorphicCache::HasEntry::offsetOfStructureID())));
    if (uid)
        slowCases.append(branchPtr(NotEqual, Address(scratch3GPR, MegamorphicCache::HasEntry::offsetOfUid()), TrustedImmPtr(uid)));
    else
        slowCases.append(branchPtr(NotEqual, Address(scratch3GPR, MegamorphicCache::HasEntry::offsetOfUid()), uidGPR));
    slowCases.append(branch32WithMemory16(NotEqual, Address(scratch3GPR, MegamorphicCache::HasEntry::offsetOfEpoch()), scratch2GPR));
    jump().linkTo(cacheHit, this);

    done.link(this);

    return slowCases;
}
#endif

AssemblyHelpers::JumpList AssemblyHelpers::loadCacheableIdentifierImpl(GPRReg propertyGPR, GPRReg destGPR, bool propertyIsString, bool propertyIsSymbol, bool canBeRope)
{
    JumpList slowCases;
    if (propertyIsString) {
        loadPtr(Address(propertyGPR, JSString::offsetOfValue()), destGPR);
        if (canBeRope)
            slowCases.append(branchIfRopeStringImpl(destGPR));
        slowCases.append(branchTest32(Zero, Address(destGPR, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIsAtom())));
    } else if (propertyIsSymbol)
        loadPtr(Address(propertyGPR, Symbol::offsetOfSymbolImpl()), destGPR);
    else {
        slowCases.append(branchIfNotCell(propertyGPR));
        auto isString = branchIfString(propertyGPR);
        slowCases.append(branchIfNotSymbol(propertyGPR));
        loadPtr(Address(propertyGPR, Symbol::offsetOfSymbolImpl()), destGPR);
        auto done = jump();

        isString.link(this);
        loadPtr(Address(propertyGPR, JSString::offsetOfValue()), destGPR);
        if (canBeRope)
            slowCases.append(branchIfRopeStringImpl(destGPR));
        slowCases.append(branchTest32(Zero, Address(destGPR, StringImpl::flagsOffset()), TrustedImm32(StringImpl::flagIsAtom())));

        done.link(this);
    }
    return slowCases;
}

void AssemblyHelpers::emitNonNullDecodeZeroExtendedStructureID(RegisterID source, RegisterID dest)
{
#if CPU(ADDRESS64)
    // This could use BFI on arm64 but that only helps if the start of structure heap is encodable as a mov and not as an immediate in the add so it's probably not super important.
    or64(TrustedImm64(structureIDBase()), source, dest);
#else // not CPU(ADDRESS64)
    move(source, dest);
#endif
}

void AssemblyHelpers::emitLoadStructure(RegisterID source, RegisterID dest)
{
    load32(MacroAssembler::Address(source, JSCell::structureIDOffset()), dest);
    emitNonNullDecodeZeroExtendedStructureID(dest, dest);
}

void AssemblyHelpers::emitLoadStructure(VM&, RegisterID source, RegisterID dest)
{
    emitLoadStructure(source, dest);
}

void AssemblyHelpers::emitEncodeStructureID(RegisterID source, RegisterID dest)
{
#if CPU(ADDRESS64)
    and64(TrustedImm32(static_cast<uint32_t>(StructureID::structureIDMask)), source, dest);
#else
    move(source, dest);
#endif
}

void AssemblyHelpers::emitLoadPrototype(VM& vm, GPRReg objectGPR, JSValueRegs resultRegs, JumpList& slowPath)
{
    ASSERT(resultRegs.payloadGPR() != objectGPR);

    slowPath.append(branchTest8(MacroAssembler::NonZero, MacroAssembler::Address(objectGPR, JSObject::typeInfoFlagsOffset()), TrustedImm32(OverridesGetPrototype)));

    emitLoadStructure(vm, objectGPR, resultRegs.payloadGPR());
    loadValue(MacroAssembler::Address(resultRegs.payloadGPR(), Structure::prototypeOffset()), resultRegs);
    auto hasMonoProto = branchIfNotEmpty(resultRegs);
    loadValue(MacroAssembler::Address(objectGPR, offsetRelativeToBase(knownPolyProtoOffset)), resultRegs);
    hasMonoProto.link(this);
}

void AssemblyHelpers::makeSpaceOnStackForCCall()
{
    unsigned stackOffset = WTF::roundUpToMultipleOf<stackAlignmentBytes()>(maxFrameExtentForSlowPathCall);
    if (stackOffset)
        subPtr(TrustedImm32(stackOffset), stackPointerRegister);
}

void AssemblyHelpers::reclaimSpaceOnStackForCCall()
{
    unsigned stackOffset = WTF::roundUpToMultipleOf<stackAlignmentBytes()>(maxFrameExtentForSlowPathCall);
    if (stackOffset)
        addPtr(TrustedImm32(stackOffset), stackPointerRegister);
}

#if USE(JSVALUE64)
template<typename LoadFromHigh, typename StoreToHigh, typename LoadFromLow, typename StoreToLow>
void emitRandomThunkImpl(AssemblyHelpers& jit, GPRReg scratch0, GPRReg scratch1, GPRReg scratch2, FPRReg result, const LoadFromHigh& loadFromHigh, const StoreToHigh& storeToHigh, const LoadFromLow& loadFromLow, const StoreToLow& storeToLow)
{
    // Inlined WeakRandom::advance().
    // uint64_t x = m_low;
    loadFromLow(scratch0);
    // uint64_t y = m_high;
    loadFromHigh(scratch1);
    // m_low = y;
    storeToLow(scratch1);

#if CPU(ARM64)
    // x ^= x << 23;
    jit.xorLeftShift64(scratch0, scratch0, AssemblyHelpers::TrustedImm32(23), scratch0);
    // x ^= x >> 17;
    jit.xorUnsignedRightShift64(scratch0, scratch0, AssemblyHelpers::TrustedImm32(17), scratch0);
    // x ^= y ^ (y >> 26);
    jit.xorUnsignedRightShift64(scratch1, scratch1, AssemblyHelpers::TrustedImm32(26), scratch2);
    jit.xor64(scratch2, scratch0);
#else
    // x ^= x << 23;
    jit.lshift64(scratch0, AssemblyHelpers::TrustedImm32(23), scratch2);
    jit.xor64(scratch2, scratch0);

    // x ^= x >> 17;
    jit.urshift64(scratch0, AssemblyHelpers::TrustedImm32(17), scratch2);
    jit.xor64(scratch2, scratch0);

    // x ^= y ^ (y >> 26);
    jit.urshift64(scratch1, AssemblyHelpers::TrustedImm32(26), scratch2);
    jit.xor64(scratch1, scratch2);
    jit.xor64(scratch2, scratch0);
#endif

    // m_high = x;
    storeToHigh(scratch0);

    // return x + y;
    jit.add64(scratch1, scratch0);

    // Extract random 53bit. [0, 53] bit is safe integer number ranges in double representation.
    jit.and64(AssemblyHelpers::TrustedImm64((1ULL << 53) - 1), scratch0);
    // Now, scratch0 is always in range of int64_t. Safe to convert it to double with cvtsi2sdq.
    jit.convertInt64ToDouble(scratch0, result);

    // Convert `(53bit double integer value) / (1 << 53)` to `(53bit double integer value) * (1.0 / (1 << 53))`.
    // In latter case, `1.0 / (1 << 53)` will become a double value represented as (mantissa = 0 & exp = 970, it means 1e-(2**54)).
    static constexpr double scale = 1.0 / (1ULL << 53);

    // Multiplying 1e-(2**54) with the double integer does not change anything of the mantissa part of the double integer.
    // It just reduces the exp part of the given 53bit double integer.
    // (Except for 0.0. This is specially handled and in this case, exp just becomes 0.)
    // Now we get 53bit precision random double value in [0, 1).
    jit.move(AssemblyHelpers::TrustedImmPtr(&scale), scratch1);
    jit.mulDouble(AssemblyHelpers::Address(scratch1), result);
}

void AssemblyHelpers::emitRandomThunk(JSGlobalObject* globalObject, GPRReg scratch0, GPRReg scratch1, GPRReg scratch2, FPRReg result)
{
    void* lowAddress = reinterpret_cast<uint8_t*>(globalObject) + JSGlobalObject::weakRandomOffset() + WeakRandom::lowOffset();
    void* highAddress = reinterpret_cast<uint8_t*>(globalObject) + JSGlobalObject::weakRandomOffset() + WeakRandom::highOffset();

    auto loadFromHigh = [&](GPRReg high) {
        load64(highAddress, high);
    };
    auto storeToHigh = [&](GPRReg high) {
        store64(high, highAddress);
    };
    auto loadFromLow = [&](GPRReg low) {
        load64(lowAddress, low);
    };
    auto storeToLow = [&](GPRReg low) {
        store64(low, lowAddress);
    };

    emitRandomThunkImpl(*this, scratch0, scratch1, scratch2, result, loadFromHigh, storeToHigh, loadFromLow, storeToLow);
}

void AssemblyHelpers::emitRandomThunk(VM& vm, GPRReg scratch0, GPRReg scratch1, GPRReg scratch2, GPRReg scratch3, FPRReg result)
{
    emitGetFromCallFrameHeaderPtr(CallFrameSlot::callee, scratch3);
    emitLoadStructure(vm, scratch3, scratch3);
    loadPtr(Address(scratch3, Structure::realmOffset()), scratch3);
    // Now, scratch3 holds JSGlobalObject*.

    auto loadFromHigh = [&](GPRReg high) {
        load64(Address(scratch3, JSGlobalObject::weakRandomOffset() + WeakRandom::highOffset()), high);
    };
    auto storeToHigh = [&](GPRReg high) {
        store64(high, Address(scratch3, JSGlobalObject::weakRandomOffset() + WeakRandom::highOffset()));
    };
    auto loadFromLow = [&](GPRReg low) {
        load64(Address(scratch3, JSGlobalObject::weakRandomOffset() + WeakRandom::lowOffset()), low);
    };
    auto storeToLow = [&](GPRReg low) {
        store64(low, Address(scratch3, JSGlobalObject::weakRandomOffset() + WeakRandom::lowOffset()));
    };

    emitRandomThunkImpl(*this, scratch0, scratch1, scratch2, result, loadFromHigh, storeToHigh, loadFromLow, storeToLow);
}
#endif

void AssemblyHelpers::emitAllocateWithNonNullAllocator(GPRReg resultGPR, const JITAllocator& allocator, GPRReg allocatorGPR, GPRReg scratchGPR, JumpList& slowPath, SlowAllocationResult slowAllocationResult)
{
    if (Options::forceGCSlowPaths()) {
        slowPath.append(jump());
        return;
    }

    // NOTE, some invariants of this function:
    // - When going to the slow path, we must leave resultGPR with zero in it.
    // - We *can not* use RegisterSet::macroScratchRegisters on x86.
    // - We *can* use RegisterSet::macroScratchRegisters on ARM.

    Jump popPath;
    Jump zeroPath;
    Jump done;

    if (allocator.isConstant())
        move(TrustedImmPtr(allocator.allocator().localAllocator()), allocatorGPR);

#if CPU(ARM) || CPU(ARM64)
    auto dataTempRegister = getCachedDataTempRegisterIDAndInvalidate();
#endif

#if CPU(ARM64)
    // On ARM64, we can leverage instructions like load-pair and shifted-add to make loading from the free list
    // and extracting interval information use less instructions.

    // Assert that we can use loadPairPtr for the interval bounds and nextInterval/secret.
    static_assert(FreeList::offsetOfIntervalEnd() - FreeList::offsetOfIntervalStart() == sizeof(uintptr_t));
    static_assert(FreeList::offsetOfNextInterval() - FreeList::offsetOfIntervalEnd() == sizeof(uintptr_t));
    static_assert(FreeList::offsetOfSecret() - FreeList::offsetOfNextInterval() == sizeof(uintptr_t));

    JIT_COMMENT(*this, "Bump allocation (fast path)");
    loadPairPtr(allocatorGPR, TrustedImm32(LocalAllocator::offsetOfFreeList() + FreeList::offsetOfIntervalStart()), resultGPR, scratchGPR);
    popPath = branchPtr(RelationalCondition::AboveOrEqual, resultGPR, scratchGPR);
    auto bumpLabel = label();
    if (allocator.isConstant())
        addPtr(TrustedImm32(allocator.allocator().cellSize()), resultGPR, scratchGPR);
    else {
        load32(Address(allocatorGPR, LocalAllocator::offsetOfCellSize()), scratchGPR);
        addPtr(resultGPR, scratchGPR);
    }
    storePtr(scratchGPR, Address(allocatorGPR, LocalAllocator::offsetOfFreeList() + FreeList::offsetOfIntervalStart()));
    done = jump();

    JIT_COMMENT(*this, "Get next interval (slower path)");
    popPath.link(this);
    loadPairPtr(allocatorGPR, TrustedImm32(LocalAllocator::offsetOfFreeList() + FreeList::offsetOfNextInterval()), resultGPR, scratchGPR);
    zeroPath = branchTestPtr(ResultCondition::NonZero, resultGPR, TrustedImm32(1));
    xor64(Address(resultGPR, FreeCell::offsetOfScrambledBits()), scratchGPR);
    addSignExtend64(resultGPR, scratchGPR, dataTempRegister);
    addUnsignedRightShift64(resultGPR, scratchGPR, TrustedImm32(32), scratchGPR);
    storePairPtr(scratchGPR, dataTempRegister, allocatorGPR, TrustedImm32(LocalAllocator::offsetOfFreeList() + FreeList::offsetOfIntervalEnd()));
    jump(bumpLabel);
#elif CPU(X86_64)
    // On x86_64, we can leverage better support for memory operands to directly interact with the free 
    // list instead of relying on registers as much.

    JIT_COMMENT(*this, "Bump allocation (fast path)");
    loadPtr(Address(allocatorGPR, LocalAllocator::offsetOfFreeList() + FreeList::offsetOfIntervalStart()), resultGPR);
    popPath = branchPtr(RelationalCondition::AboveOrEqual, resultGPR, Address(allocatorGPR, LocalAllocator::offsetOfFreeList() + FreeList::offsetOfIntervalEnd()));
    auto bumpLabel = label();
    if (allocator.isConstant())
        add64(TrustedImm32(allocator.allocator().cellSize()), Address(allocatorGPR, LocalAllocator::offsetOfFreeList() + FreeList::offsetOfIntervalStart()));
    else {
        load32(Address(allocatorGPR, LocalAllocator::offsetOfCellSize()), scratchGPR);
        add64(scratchGPR, Address(allocatorGPR, LocalAllocator::offsetOfFreeList() + FreeList::offsetOfIntervalStart()));
    }
    done = jump();

    JIT_COMMENT(*this, "Get next interval (slower path)");
    popPath.link(this);
    loadPtr(Address(allocatorGPR, LocalAllocator::offsetOfFreeList() + FreeList::offsetOfNextInterval()), resultGPR);
    zeroPath = branchTestPtr(ResultCondition::NonZero, resultGPR, TrustedImm32(1));
    load32(Address(allocatorGPR, LocalAllocator::offsetOfFreeList() + FreeList::offsetOfSecret()), scratchGPR);
    xor32(Address(resultGPR, FreeCell::offsetOfScrambledBits()), scratchGPR); // Lower 32 bits -> offset to next interval
    add64(scratchGPR, Address(allocatorGPR, LocalAllocator::offsetOfFreeList() + FreeList::offsetOfNextInterval()));
    load32(Address(allocatorGPR, LocalAllocator::offsetOfFreeList() + FreeList::offsetOfSecret() + 4), scratchGPR);
    xor32(Address(resultGPR, FreeCell::offsetOfScrambledBits() + 4), scratchGPR); // Upper 32 bits -> size of interval
    storePtr(resultGPR, Address(allocatorGPR, LocalAllocator::offsetOfFreeList() + FreeList::offsetOfIntervalStart()));
    addPtr(resultGPR, scratchGPR);
    storePtr(scratchGPR, Address(allocatorGPR, LocalAllocator::offsetOfFreeList() + FreeList::offsetOfIntervalEnd()));
    jump(bumpLabel);
#else
    // Otherwise, we have a fairly general case for all other architectures here.

    // Bump allocation (fast path)
    loadPtr(Address(allocatorGPR, LocalAllocator::offsetOfFreeList() + FreeList::offsetOfIntervalStart()), resultGPR);
    loadPtr(Address(allocatorGPR, LocalAllocator::offsetOfFreeList() + FreeList::offsetOfIntervalEnd()), scratchGPR);
    popPath = branchPtr(RelationalCondition::AboveOrEqual, resultGPR, scratchGPR);
    auto bumpLabel = label();
    if (allocator.isConstant()) {
        move(resultGPR, scratchGPR);
        addPtr(TrustedImm32(allocator.allocator().cellSize()), scratchGPR);
    } else {
        load32(Address(allocatorGPR, LocalAllocator::offsetOfCellSize()), scratchGPR);
        addPtr(resultGPR, scratchGPR);
    }
    storePtr(scratchGPR, Address(allocatorGPR, LocalAllocator::offsetOfFreeList() + FreeList::offsetOfIntervalStart()));
    done = jump();

    // Get next interval (slower path)
    popPath.link(this);
    loadPtr(Address(allocatorGPR, LocalAllocator::offsetOfFreeList() + FreeList::offsetOfNextInterval()), resultGPR);
    zeroPath = branchTestPtr(ResultCondition::NonZero, resultGPR, TrustedImm32(1));
    load32(Address(allocatorGPR, LocalAllocator::offsetOfFreeList() + FreeList::offsetOfSecret()), scratchGPR);
    xor32(Address(resultGPR, FreeCell::offsetOfScrambledBits()), scratchGPR); // Lower 32 bits -> offset to next interval
    loadPtr(Address(allocatorGPR, LocalAllocator::offsetOfFreeList() + FreeList::offsetOfNextInterval()), dataTempRegister);
    addPtr(scratchGPR, dataTempRegister);
    storePtr(dataTempRegister, Address(allocatorGPR, LocalAllocator::offsetOfFreeList() + FreeList::offsetOfNextInterval()));
    load32(Address(allocatorGPR, LocalAllocator::offsetOfFreeList() + FreeList::offsetOfSecret() + 4), scratchGPR);
    xor32(Address(resultGPR, FreeCell::offsetOfScrambledBits() + 4), scratchGPR); // Upper 32 bits -> size of interval
    addPtr(resultGPR, scratchGPR);
    storePtr(scratchGPR, Address(allocatorGPR, LocalAllocator::offsetOfFreeList() + FreeList::offsetOfIntervalEnd()));
    jump(bumpLabel);
#endif

    if (slowAllocationResult == SlowAllocationResult::ClearToNull) {
        zeroPath.link(this);
        move(TrustedImm32(0), resultGPR);
        slowPath.append(jump());
    } else
        slowPath.append(zeroPath);

    done.link(this);
}

void AssemblyHelpers::emitLoadTLCAllocatorForSlot(GPRReg allocatorGPR, unsigned tlcSlot, JumpList& slowPath)
{
    // H-VMLITE-TLCPTR (SPEC-heap §5.3/§B.4; GILOFF-TAX #1/#2): the §B.4
    // addressing chain collapsed to the lite mirror —
    //   lite = loadVMLite
    //   slot < [lite + offsetOfTlcTableBound] ? : slowPath   (32-bit)
    //   table = [lite + offsetOfTlcTable]
    //   allocatorGPR = [table + slot * sizeof(Allocator)]    (LocalAllocator* | null)
    // The bound is stamped LAST (GCThreadLocalCache::growTable / the §10A.1
    // attach stamp) and grow-only, so passing the bound check guarantees both
    // table != null and table[slot] is an initialized Allocator word; a stale
    // (smaller) bound only over-dispatches to the slow path. Same-thread (I11
    // reads what I2 wrote) — no fences emitted. Caller passes the result
    // through JITAllocator::variable() so emitAllocate adds the null check.
    static_assert(sizeof(Allocator) == sizeof(void*));
    JIT_COMMENT(*this, "TLC lite-relative allocator for baked slot ", tlcSlot);
    loadVMLite(allocatorGPR);
    slowPath.append(branch32(BelowOrEqual, Address(allocatorGPR, static_cast<int32_t>(VMLite::offsetOfTlcTableBound())), TrustedImm32(static_cast<int32_t>(tlcSlot))));
    loadPtr(Address(allocatorGPR, static_cast<int32_t>(VMLite::offsetOfTlcTable())), allocatorGPR);
    loadPtr(Address(allocatorGPR, static_cast<int32_t>(tlcSlot * sizeof(void*))), allocatorGPR);
}

void AssemblyHelpers::emitAllocate(GPRReg resultGPR, const JITAllocator& allocator, GPRReg allocatorGPR, GPRReg scratchGPR, JumpList& slowPath, SlowAllocationResult slowAllocationResult)
{
    switch (allocator.kind()) {
    case JITAllocator::Constant:
        if (!allocator.allocator()) {
            slowPath.append(jump());
            return;
        }
        break;
    case JITAllocator::Variable:
        slowPath.append(branchTestPtr(Zero, allocatorGPR));
        break;
    case JITAllocator::VariableNonNull:
        break;
    }

    emitAllocateWithNonNullAllocator(resultGPR, allocator, allocatorGPR, scratchGPR, slowPath, slowAllocationResult);
}

void AssemblyHelpers::emitAllocateVariableSized(GPRReg resultGPR, const JITAllocator& allocator, Address subspaceAllocatorsBase, GPRReg allocationSize, GPRReg scratchGPR1, GPRReg scratchGPR2, JumpList& slowPath, SlowAllocationResult slowAllocationResult)
{
    static_assert(!(MarkedSpace::sizeStep & (MarkedSpace::sizeStep - 1)), "MarkedSpace::sizeStep must be a power of two.");

    unsigned stepShift = getLSBSet(MarkedSpace::sizeStep);

    addPtr(TrustedImm32(MarkedSpace::sizeStep - 1), allocationSize, scratchGPR1);
    urshiftPtr(TrustedImm32(stepShift), scratchGPR1);
    slowPath.append(branchPtr(Above, scratchGPR1, TrustedImmPtr(MarkedSpace::largeCutoff >> stepShift)));
    JIT_COMMENT(*this, "Load the Allocator from the CompleteSubspace's buffer");
    loadPtr(subspaceAllocatorsBase.indexedBy(scratchGPR1, ScalePtr), scratchGPR1);

    emitAllocate(resultGPR, allocator, scratchGPR1, scratchGPR2, slowPath, slowAllocationResult);
}

void AssemblyHelpers::emitAllocateVariableSized(GPRReg resultGPR, CompleteSubspace& subspace, GPRReg allocationSize, GPRReg scratchGPR1, GPRReg scratchGPR2, JumpList& slowPath, SlowAllocationResult slowAllocationResult)
{
    JIT_COMMENT(*this, "Materialize Allocator buffer base");
    move(TrustedImmPtr(subspace.allocatorsForSizeSteps().data()), scratchGPR2);
    emitAllocateVariableSized(resultGPR, JITAllocator::variable(), Address(scratchGPR2, 0), allocationSize, scratchGPR1, scratchGPR2, slowPath, slowAllocationResult);
}

void AssemblyHelpers::restoreCalleeSavesFromEntryFrameCalleeSavesBuffer(EntryFrame*& topEntryFrame)
{
#if NUMBER_OF_CALLEE_SAVES_REGISTERS > 0
    RegisterAtOffsetList* allCalleeSaves = RegisterSet::vmCalleeSaveRegisterOffsets();
    auto dontRestoreRegisters = RegisterSet::stackRegisters();
    unsigned registerCount = allCalleeSaves->registerCount();
    if constexpr (AssemblyHelpersInternal::dumpVerbose)
        JIT_COMMENT(*this, "restoreCalleeSavesFromEntryFrameCalleeSavesBuffer ", *allCalleeSaves, " skip: ", dontRestoreRegisters);
    else
        JIT_COMMENT(*this, "restoreCalleeSavesFromEntryFrameCalleeSavesBuffer");

    GPRReg scratch = InvalidGPRReg;
    unsigned scratchGPREntryIndex = 0;
#if CPU(ARM64)
    // We don't need a second scratch GPR, but we'll also defer restoring this
    // GPR (in the next slot after the scratch) so that we can restore them together
    // later using a loadPair64.
    GPRReg unusedNextSlotGPR = InvalidGPRReg;
#endif

    // Use the first GPR entry's register as our baseGPR.
    for (unsigned i = 0; i < registerCount; i++) {
        RegisterAtOffset entry = allCalleeSaves->at(i);
        if (dontRestoreRegisters.contains(entry.reg(), IgnoreVectors))
            continue;
        if (entry.reg().isGPR()) {
#if CPU(ARM64)
            if (i + 1 < registerCount) {
                RegisterAtOffset entry2 = allCalleeSaves->at(i + 1);
                if (!dontRestoreRegisters.contains(entry2.reg(), IgnoreVectors)
                    && entry2.reg().isGPR()
                    && entry2.offset() == entry.offset() + static_cast<ptrdiff_t>(sizeof(CPURegister))) {
                    scratchGPREntryIndex = i;
                    scratch = entry.reg().gpr();
                    unusedNextSlotGPR = entry2.reg().gpr();
                    break;
                }
            }
#else
            scratchGPREntryIndex = i;
            scratch = entry.reg().gpr();
            break;
#endif
        }
    }
    ASSERT(scratch != InvalidGPRReg);

    RegisterSet skipList;
    skipList.merge(dontRestoreRegisters);

    // Skip the scratch register(s). We'll restore them later.
    skipList.add(scratch, IgnoreVectors);
#if CPU(ARM64)
    RELEASE_ASSERT(unusedNextSlotGPR != InvalidGPRReg);
    skipList.add(unusedNextSlotGPR, IgnoreVectors);
#endif

    loadPtr(&topEntryFrame, scratch);
    restoreCalleeSavesFromVMEntryFrameCalleeSavesBufferImpl(scratch, skipList);

    // Restore the callee save value of the scratch.
    RegisterAtOffset entry = allCalleeSaves->at(scratchGPREntryIndex);
    ASSERT(!dontRestoreRegisters.contains(entry.reg(), IgnoreVectors));
    ASSERT(entry.reg().isGPR());
    ASSERT(scratch == entry.reg().gpr());
#if CPU(ARM64)
    RegisterAtOffset entry2 = allCalleeSaves->at(scratchGPREntryIndex + 1);
    ASSERT_UNUSED(entry2, !dontRestoreRegisters.contains(entry2.reg(), IgnoreVectors));
    ASSERT(entry2.reg().isGPR());
    ASSERT(unusedNextSlotGPR == entry2.reg().gpr());
    loadPair64(scratch, TrustedImm32(entry.offset()), scratch, unusedNextSlotGPR);
#else
    loadPtr(Address(scratch, entry.offset()), scratch);
#endif

#else
    UNUSED_PARAM(topEntryFrame);
#endif // NUMBER_OF_CALLEE_SAVES_REGISTERS > 0
}

void AssemblyHelpers::restoreCalleeSavesFromVMEntryFrameCalleeSavesBuffer(GPRReg vmGPR, GPRReg scratchGPR)
{
#if NUMBER_OF_CALLEE_SAVES_REGISTERS > 0
    loadPtr(Address(vmGPR, VM::topEntryFrameOffset()), scratchGPR);
    restoreCalleeSavesFromVMEntryFrameCalleeSavesBufferImpl(scratchGPR, RegisterSet::stackRegisters());
#else
    UNUSED_PARAM(vmGPR);
#endif // NUMBER_OF_CALLEE_SAVES_REGISTERS > 0
}

void AssemblyHelpers::restoreCalleeSavesFromVMEntryFrameCalleeSavesBufferImpl(GPRReg entryFrameGPR, const RegisterSet& skipList)
{
#if NUMBER_OF_CALLEE_SAVES_REGISTERS > 0
    addPtr(TrustedImm32(EntryFrame::calleeSaveRegistersBufferOffset()), entryFrameGPR);

    RegisterAtOffsetList* allCalleeSaves = RegisterSet::vmCalleeSaveRegisterOffsets();
    if constexpr (AssemblyHelpersInternal::dumpVerbose)
        JIT_COMMENT(*this, "restoreCalleeSavesFromVMEntryFrameCalleeSavesBufferImpl ", entryFrameGPR, " callee saves: ", *allCalleeSaves, " skip: ", skipList);
    else
        JIT_COMMENT(*this, "restoreCalleeSavesFromVMEntryFrameCalleeSavesBufferImpl");
    unsigned registerCount = allCalleeSaves->registerCount();

    LoadRegSpooler spooler(*this, entryFrameGPR);

    // Restore all callee saves except for the scratch.
    unsigned i = 0;
    for (; i < registerCount; i++) {
        RegisterAtOffset entry = allCalleeSaves->at(i);
        if (skipList.contains(entry.reg(), IgnoreVectors))
            continue;
        if (!entry.reg().isGPR())
            break;
        spooler.loadGPR(entry);
    }
    spooler.finalizeGPR();
    for (; i < registerCount; i++) {
        RegisterAtOffset entry = allCalleeSaves->at(i);
        if (skipList.contains(entry.reg(), IgnoreVectors))
            continue;
        ASSERT(!entry.reg().isGPR());
        spooler.loadFPR(entry);
    }
    spooler.finalizeFPR();

#else
    UNUSED_PARAM(vmGPR);
    UNUSED_PARAM(skipList);
#endif // NUMBER_OF_CALLEE_SAVES_REGISTERS > 0
}

void AssemblyHelpers::emitVirtualCall(VM& vm, CallLinkInfo* info)
{
    move(TrustedImmPtr(info), GPRInfo::regT2);
    emitVirtualCallWithoutMovingGlobalObject(vm, GPRInfo::regT2, info->callMode());
}

void AssemblyHelpers::emitVirtualCallWithoutMovingGlobalObject(VM& vm, GPRReg callLinkInfoGPR, CallMode callMode)
{
    move(callLinkInfoGPR, GPRInfo::regT2);
    nearCallThunk(CodeLocationLabel<JITStubRoutinePtrTag> { vm.getCTIVirtualCall(callMode).code() });
}

#if USE(JSVALUE64)
void AssemblyHelpers::rapidHashMix64(GPRReg inputAndResult, GPRReg scratch1, GPRReg scratch2)
{
    // rapidhash "mum" mixer. Keep in sync with WTF intHash and FTL rapidHashMix64 code
    // as we need to use the same hash function.
    //
    //   a = key ^ secret1
    //   b = key ^ secret2
    //   product = a * b                  (128-bit)
    //   result  = lo64(product) ^ hi64(product)
    //
    GPRReg input = inputAndResult;

    ASSERT(scratch1 != scratch2);
    ASSERT(scratch1 != input);
    ASSERT(scratch2 != input);

#if CPU(ARM64)
    // scratch1 = input ^ secret1 = a
    move(TrustedImm64(static_cast<int64_t>(0x2d358dccaa6c78a5ULL)), scratch1);
    xor64(input, scratch1);
    // scratch2 = input ^ secret2 = b
    move(TrustedImm64(static_cast<int64_t>(0x8bb84b93962eacc9ULL)), scratch2);
    xor64(input, scratch2);
    // input is dead as a value now; reuse its register for the hi half.
    uMulHigh64(scratch1, scratch2, input);  // input = hi64(a * b)
    mul64(scratch1, scratch2, scratch1);    // scratch1 = lo64(a * b); scratch2 preserved
    xor64(scratch1, input);                 // input = lo ^ hi = hash
#elif CPU(X86_64)
    // x86-64 mulq uses rax and rdx implicitly: rdx:rax = rax * src.
    // We use scratchRegister, scratch1, and scratch2 to save and restore rax / rdx.
    GPRReg inputSafe = scratchRegister();
    move(input, inputSafe); // scratchRegister is never rax / rdx.

    // Whether we need to preserve either rax or rdx.
    // If they are in scratches, then we do not need to specially save and restore them.
    bool preserveRax = (scratch1 != X86Registers::eax) && (scratch2 != X86Registers::eax);
    bool preserveRdx = (scratch1 != X86Registers::edx) && (scratch2 != X86Registers::edx);

    // A scratch is a valid persistent save slot only if it is neither rax nor rdx
    GPRReg safeScratch1 = (scratch1 != X86Registers::eax && scratch1 != X86Registers::edx) ? scratch1 : InvalidGPRReg;
    GPRReg safeScratch2 = (scratch2 != X86Registers::eax && scratch2 != X86Registers::edx) ? scratch2 : InvalidGPRReg;

    GPRReg raxSaveSlot = InvalidGPRReg;
    GPRReg rdxSaveSlot = InvalidGPRReg;
    if (preserveRax) {
        raxSaveSlot = (safeScratch1 != InvalidGPRReg) ? safeScratch1 : safeScratch2;
        ASSERT(raxSaveSlot != InvalidGPRReg);
    }
    if (preserveRdx) {
        if (safeScratch1 != InvalidGPRReg && safeScratch1 != raxSaveSlot)
            rdxSaveSlot = safeScratch1;
        else
            rdxSaveSlot = safeScratch2;
        ASSERT(rdxSaveSlot != InvalidGPRReg);
    }

    if (preserveRax)
        move(X86Registers::eax, raxSaveSlot);
    if (preserveRdx)
        move(X86Registers::edx, rdxSaveSlot);

    // rax = input ^ secret1 = a
    move(TrustedImm64(static_cast<int64_t>(0x2d358dccaa6c78a5ULL)), X86Registers::eax);
    xor64(inputSafe, X86Registers::eax);
    // rdx = input ^ secret2 = b
    move(TrustedImm64(static_cast<int64_t>(0x8bb84b93962eacc9ULL)), X86Registers::edx);
    xor64(inputSafe, X86Registers::edx);

    // mulq rdx: rdx:rax = rax * rdx = a * b; rax has lo, rdx has hi.
    m_assembler.mulq_r(X86Registers::edx);
    xor64(X86Registers::edx, X86Registers::eax);  // rax = lo ^ hi = hash

    move(X86Registers::eax, scratchRegister());

    if (preserveRax)
        move(raxSaveSlot, X86Registers::eax);
    if (preserveRdx)
        move(rdxSaveSlot, X86Registers::edx);

    move(scratchRegister(), inputAndResult);
#else
    UNUSED_PARAM(scratch1);
    UNUSED_PARAM(scratch2);
    RELEASE_ASSERT_NOT_REACHED();
#endif

    // return static_cast<unsigned>(result)
    zeroExtend32ToWord(inputAndResult, inputAndResult);
}
#endif // USE(JSVALUE64)

void AssemblyHelpers::emitConvertValueToBoolean(VM& vm, JSValueRegs value, GPRReg result, GPRReg scratchIfShouldCheckMasqueradesAsUndefined, FPRReg valueAsFPR, FPRReg tempFPR, bool shouldCheckMasqueradesAsUndefined, JSGlobalObject* globalObject, bool invert)
{
    // Implements the following control flow structure:
    // if (value is cell) {
    //     if (value is string or value is HeapBigInt)
    //         result = !!value->length
    //     else {
    //         do evil things for masquerades-as-undefined
    //         result = true
    //     }
    // } else if (value is int32) {
    //     result = !!unboxInt32(value)
    // } else if (value is number) {
    //     result = !!unboxDouble(value)
    // } else if (value is BigInt32) {
    //     result = !!unboxBigInt32(value)
    // } else {
    //     result = value == jsTrue
    // }

    JumpList done;

    auto notCell = branchIfNotCell(value);
    auto isString = branchIfString(value.payloadGPR());
    auto isHeapBigInt = branchIfHeapBigInt(value.payloadGPR());

    if (shouldCheckMasqueradesAsUndefined) {
        ASSERT(scratchIfShouldCheckMasqueradesAsUndefined != InvalidGPRReg);
        JumpList isNotMasqueradesAsUndefined;
        isNotMasqueradesAsUndefined.append(branchTest8(Zero, Address(value.payloadGPR(), JSCell::typeInfoFlagsOffset()), TrustedImm32(MasqueradesAsUndefined)));
        emitLoadStructure(vm, value.payloadGPR(), result);
        move(TrustedImmPtr(globalObject), scratchIfShouldCheckMasqueradesAsUndefined);
        isNotMasqueradesAsUndefined.append(branchPtr(NotEqual, Address(result, Structure::realmOffset()), scratchIfShouldCheckMasqueradesAsUndefined));

        // We act like we are "undefined" here.
        move(invert ? TrustedImm32(1) : TrustedImm32(0), result);
        done.append(jump());
        isNotMasqueradesAsUndefined.link(this);
    }
    move(invert ? TrustedImm32(0) : TrustedImm32(1), result);
    done.append(jump());

    isString.link(this);
    move(TrustedImmPtr(jsEmptyString(vm)), result);
    comparePtr(invert ? Equal : NotEqual, value.payloadGPR(), result, result);
    done.append(jump());

    isHeapBigInt.link(this);
    load32(Address(value.payloadGPR(), JSBigInt::offsetOfLength()), result);
    compare32(invert ? Equal : NotEqual, result, TrustedImm32(0), result);
    done.append(jump());

    notCell.link(this);
    auto notInt32 = branchIfNotInt32(value);
    compare32(invert ? Equal : NotEqual, value.payloadGPR(), TrustedImm32(0), result);
    done.append(jump());

    notInt32.link(this);
    auto notDouble = branchIfNotDoubleKnownNotInt32(value);
#if USE(JSVALUE64)
    unboxDouble(value.gpr(), result, valueAsFPR);
#else
    unboxDouble(value, valueAsFPR);
#endif
    move(invert ? TrustedImm32(1) : TrustedImm32(0), result);
    done.append(branchDoubleZeroOrNaN(valueAsFPR, tempFPR));
    move(invert ? TrustedImm32(0) : TrustedImm32(1), result);
    done.append(jump());

    notDouble.link(this);
#if USE(BIGINT32)
    auto isNotBigInt32 = branchIfNotBigInt32(value.gpr(), result);
    move(value.gpr(), result);
    urshift64(TrustedImm32(16), result);
    compare32(invert ? Equal : NotEqual, result, TrustedImm32(0), result);
    done.append(jump());

    isNotBigInt32.link(this);
#endif // USE(BIGINT32)
#if USE(JSVALUE64)
    compare64(invert ? NotEqual : Equal, value.gpr(), TrustedImm32(JSValue::ValueTrue), result);
#else
    move(invert ? TrustedImm32(1) : TrustedImm32(0), result);
    done.append(branchIfNotBoolean(value, InvalidGPRReg));
    compare32(invert ? Equal : NotEqual, value.payloadGPR(), TrustedImm32(0), result);
#endif

    done.link(this);
}

AssemblyHelpers::JumpList AssemblyHelpers::branchIfValue(VM& vm, JSValueRegs value, GPRReg scratch, GPRReg scratchIfShouldCheckMasqueradesAsUndefined, FPRReg valueAsFPR, FPRReg tempFPR, bool shouldCheckMasqueradesAsUndefined, Variant<JSGlobalObject*, GPRReg, LazyGlobalObjectLoadTag> globalObject, bool invert)
{
    // Implements the following control flow structure:
    // if (value is cell) {
    //     if (value is string or value is HeapBigInt)
    //         result = !!value->length
    //     else {
    //         do evil things for masquerades-as-undefined
    //         result = true
    //     }
    // } else if (value is int32) {
    //     result = !!unboxInt32(value)
    // } else if (value is number) {
    //     result = !!unboxDouble(value)
    // } else if (value is BigInt32) {
    //     result = !!unboxBigInt32(value)
    // } else {
    //     result = value == jsTrue
    // }

    JumpList done;
    JumpList truthy;

    auto notCell = branchIfNotCell(value);
    auto isString = branchIfString(value.payloadGPR());
    auto isHeapBigInt = branchIfHeapBigInt(value.payloadGPR());

    if (shouldCheckMasqueradesAsUndefined) {
        ASSERT(scratchIfShouldCheckMasqueradesAsUndefined != InvalidGPRReg);
        JumpList isNotMasqueradesAsUndefined;
        isNotMasqueradesAsUndefined.append(branchTest8(Zero, Address(value.payloadGPR(), JSCell::typeInfoFlagsOffset()), TrustedImm32(MasqueradesAsUndefined)));
        emitLoadStructure(vm, value.payloadGPR(), scratch);
        if (std::holds_alternative<JSGlobalObject*>(globalObject))
            move(TrustedImmPtr(std::get<JSGlobalObject*>(globalObject)), scratchIfShouldCheckMasqueradesAsUndefined);
        else if (std::holds_alternative<GPRReg>(globalObject))
            move(std::get<GPRReg>(globalObject), scratchIfShouldCheckMasqueradesAsUndefined);
        else
            loadPtr(Address(GPRInfo::jitDataRegister, BaselineJITData::offsetOfGlobalObject()), scratchIfShouldCheckMasqueradesAsUndefined);
        isNotMasqueradesAsUndefined.append(branchPtr(NotEqual, Address(scratch, Structure::realmOffset()), scratchIfShouldCheckMasqueradesAsUndefined));

        // We act like we are "undefined" here.
        if (invert)
            truthy.append(jump());
        else
            done.append(jump());

        if (invert)
            done.append(isNotMasqueradesAsUndefined);
        else
            truthy.append(isNotMasqueradesAsUndefined);
    } else {
        if (invert)
            done.append(jump());
        else
            truthy.append(jump());
    }

    isString.link(this);
    truthy.append(branchPtr(invert ? Equal : NotEqual, value.payloadGPR(), TrustedImmPtr(jsEmptyString(vm))));
    done.append(jump());

    isHeapBigInt.link(this);
    truthy.append(branchTest32(invert ? Zero : NonZero, Address(value.payloadGPR(), JSBigInt::offsetOfLength())));
    done.append(jump());

    notCell.link(this);
    auto notInt32 = branchIfNotInt32(value);
    truthy.append(branchTest32(invert ? Zero : NonZero, value.payloadGPR()));
    done.append(jump());

    notInt32.link(this);
    auto notDouble = branchIfNotDoubleKnownNotInt32(value);
#if USE(JSVALUE64)
    unboxDouble(value.gpr(), scratch, valueAsFPR);
#else
    unboxDouble(value, valueAsFPR);
#endif
    if (invert) {
        truthy.append(branchDoubleZeroOrNaN(valueAsFPR, tempFPR));
        done.append(jump());
    } else {
        done.append(branchDoubleZeroOrNaN(valueAsFPR, tempFPR));
        truthy.append(jump());
    }

    notDouble.link(this);
#if USE(BIGINT32)
    auto isNotBigInt32 = branchIfNotBigInt32(value.gpr(), scratch);
    move(value.gpr(), scratch);
    urshift64(TrustedImm32(16), scratch);
    truthy.append(branchTest32(invert ? Zero : NonZero, scratch));
    done.append(jump());

    isNotBigInt32.link(this);
#endif // USE(BIGINT32)
#if USE(JSVALUE64)
    truthy.append(branch64(invert ? NotEqual : Equal, value.gpr(), TrustedImm64(JSValue::encode(jsBoolean(true)))));
#else
    auto notBoolean = branchIfNotBoolean(value, InvalidGPRReg);
    if (invert)
        truthy.append(notBoolean);
    else
        done.append(notBoolean);
    truthy.append(branch32(invert ? Equal : NotEqual, value.payloadGPR(), TrustedImm32(0)));
#endif

    done.link(this);

    return truthy;
}

#if ENABLE(WEBASSEMBLY)
void AssemblyHelpers::storeWasmContextInstance(GPRReg src)
{
    JIT_COMMENT(*this, "Store wasm context instance from", src);
    move(src, GPRInfo::wasmContextInstancePointer);
    JIT_COMMENT(*this, "Store wasm context instance done");
}

void AssemblyHelpers::prepareWasmCallOperation(GPRReg instanceGPR)
{
    UNUSED_PARAM(instanceGPR);
#if !USE(BUILTIN_FRAME_ADDRESS) || ASSERT_ENABLED
    storePtr(GPRInfo::callFrameRegister, Address(instanceGPR, JSWebAssemblyInstance::offsetOfTemporaryCallFrame()));
#endif
}

#endif // ENABLE(WEBASSEMBLY)

void AssemblyHelpers::copyCalleeSavesToEntryFrameCalleeSavesBufferImpl(GPRReg calleeSavesBuffer)
{
    JIT_COMMENT(*this, "copyCalleeSavesToEntryFrameCalleeSavesBufferImpl ", calleeSavesBuffer);
#if NUMBER_OF_CALLEE_SAVES_REGISTERS > 0
    addPtr(TrustedImm32(EntryFrame::calleeSaveRegistersBufferOffset()), calleeSavesBuffer);

    RegisterAtOffsetList* allCalleeSaves = RegisterSet::vmCalleeSaveRegisterOffsets();
    auto dontCopyRegisters = RegisterSet::stackRegisters();
    unsigned registerCount = allCalleeSaves->registerCount();

    StoreRegSpooler spooler(*this, calleeSavesBuffer);

    unsigned i = 0;
    for (; i < registerCount; i++) {
        RegisterAtOffset entry = allCalleeSaves->at(i);
        if (dontCopyRegisters.contains(entry.reg(), IgnoreVectors))
            continue;
        if (!entry.reg().isGPR())
            break;
        spooler.storeGPR(entry);
    }
    spooler.finalizeGPR();
    for (; i < registerCount; i++) {
        RegisterAtOffset entry = allCalleeSaves->at(i);
        if (dontCopyRegisters.contains(entry.reg(), IgnoreVectors))
            continue;
        spooler.storeFPR(entry);
    }
    spooler.finalizeFPR();

#else
    UNUSED_PARAM(calleeSavesBuffer);
#endif
}

void AssemblyHelpers::cage(Gigacage::Kind kind, GPRReg storage)
{
#if GIGACAGE_ENABLED
    if (!Gigacage::isEnabled(kind))
        return;
    andPtr(TrustedImmPtr(Gigacage::mask(kind)), storage);
    addPtr(TrustedImmPtr(Gigacage::basePtr(kind)), storage);
#else
    UNUSED_PARAM(kind);
    UNUSED_PARAM(storage);
#endif
}

// length may be the same register as scratch.
void AssemblyHelpers::cageConditionally(Gigacage::Kind kind, GPRReg storage, GPRReg length, GPRReg scratch)
{
#if GIGACAGE_ENABLED
    if (Gigacage::isEnabled(kind)) {
        if (kind != Gigacage::Primitive || Gigacage::disablingPrimitiveGigacageIsForbidden())
            cage(kind, storage);
        else {
            JumpList done;
            done.append(branchTest8(NonZero, AbsoluteAddress(&Gigacage::disablePrimitiveGigacageRequested)));

            loadPtr(Gigacage::addressOfBasePtr(kind), scratch);
            done.append(branchTest64(Zero, scratch));
            andPtr(TrustedImmPtr(Gigacage::mask(kind)), storage);
            addPtr(scratch, storage);
            done.link(this);
        }
    }
#endif

    UNUSED_PARAM(kind);
    UNUSED_PARAM(storage);
    UNUSED_PARAM(length);
    UNUSED_PARAM(scratch);
}

void AssemblyHelpers::emitSave(const RegisterAtOffsetList& list)
{
    if constexpr (AssemblyHelpersInternal::dumpVerbose)
        JIT_COMMENT(*this, "emitSave ", list);
    StoreRegSpooler spooler(*this, framePointerRegister);

    size_t registerCount = list.registerCount();
    size_t i = 0;
    for (; i < registerCount; i++) {
        auto entry = list.at(i);
        if (!entry.reg().isGPR())
            break;
        spooler.storeGPR(entry);
    }
    spooler.finalizeGPR();

    for (; i < registerCount; i++)
        spooler.storeFPR(list.at(i));
    spooler.finalizeFPR();
}

void AssemblyHelpers::emitRestore(const RegisterAtOffsetList& list, GPRReg baseGPR)
{
    if constexpr (AssemblyHelpersInternal::dumpVerbose)
        JIT_COMMENT(*this, "emitRestore ", list);
    LoadRegSpooler spooler(*this, baseGPR);

    size_t registerCount = list.registerCount();
    size_t i = 0;
    for (; i < registerCount; i++) {
        auto entry = list.at(i);
        if (!entry.reg().isGPR())
            break;
        spooler.loadGPR(entry);
    }
    spooler.finalizeGPR();

    for (; i < registerCount; i++)
        spooler.loadFPR(list.at(i));
    spooler.finalizeFPR();
}

void AssemblyHelpers::emitSaveCalleeSavesFor(const RegisterAtOffsetList* calleeSaves)
{
    auto dontSaveRegisters = RegisterSet::stackRegisters();
    unsigned registerCount = calleeSaves->registerCount();
    if constexpr (AssemblyHelpersInternal::dumpVerbose)
        JIT_COMMENT(*this, "emitSaveCalleeSavesFor ", *calleeSaves, " dontRestore: ", dontSaveRegisters);
    else
        JIT_COMMENT(*this, "emitSaveCalleeSavesFor");

    StoreRegSpooler spooler(*this, framePointerRegister);

    unsigned i = 0;
    for (; i < registerCount; i++) {
        RegisterAtOffset entry = calleeSaves->at(i);
        if (entry.reg().isFPR())
            break;
        if (dontSaveRegisters.contains(entry.reg(), IgnoreVectors))
            continue;
        spooler.storeGPR(entry);
    }
    spooler.finalizeGPR();
    for (; i < registerCount; i++) {
        RegisterAtOffset entry = calleeSaves->at(i);
        if (dontSaveRegisters.contains(entry.reg(), IgnoreVectors))
            continue;
        spooler.storeFPR(entry);
    }
    spooler.finalizeFPR();
}

void AssemblyHelpers::emitRestoreCalleeSavesFor(const RegisterAtOffsetList* calleeSaves)
{
    auto dontRestoreRegisters = RegisterSet::stackRegisters();
    unsigned registerCount = calleeSaves->registerCount();
    if constexpr (AssemblyHelpersInternal::dumpVerbose)
        JIT_COMMENT(*this, "emitRestoreCalleeSavesFor ", *calleeSaves, " dontSave: ", dontRestoreRegisters);
    else
        JIT_COMMENT(*this, "emitRestoreCalleeSavesFor");

    LoadRegSpooler spooler(*this, framePointerRegister);

    unsigned i = 0;
    for (; i < registerCount; i++) {
        RegisterAtOffset entry = calleeSaves->at(i);
        if (entry.reg().isFPR())
            break;
        if (dontRestoreRegisters.contains(entry.reg(), IgnoreVectors))
            continue;
        spooler.loadGPR(entry);
    }
    spooler.finalizeGPR();
    for (; i < registerCount; i++) {
        RegisterAtOffset entry = calleeSaves->at(i);
        if (dontRestoreRegisters.contains(entry.reg(), IgnoreVectors))
            continue;
        spooler.loadFPR(entry);
    }
    spooler.finalizeFPR();
}

void AssemblyHelpers::copyLLIntBaselineCalleeSavesFromFrameOrRegisterToEntryFrameCalleeSavesBuffer(EntryFrame*& topEntryFrame, const RegisterSet& usedRegisters)
{
#if NUMBER_OF_CALLEE_SAVES_REGISTERS > 0
    // Copy saved calleeSaves on stack or unsaved calleeSaves in register to vm calleeSave buffer
    ScratchRegisterAllocator allocator(usedRegisters);
    GPRReg destBufferGPR = allocator.allocateScratchGPR();
    GPRReg temp1 = allocator.allocateScratchGPR();
    FPRReg fpTemp1 = allocator.allocateScratchFPR();
    GPRReg temp2 = allocator.allocateScratchGPR();
    FPRReg fpTemp2 = allocator.allocateScratchFPR();
    RELEASE_ASSERT(!allocator.didReuseRegisters());

    loadPtr(&topEntryFrame, destBufferGPR);
    addPtr(TrustedImm32(EntryFrame::calleeSaveRegistersBufferOffset()), destBufferGPR);

    CopySpooler spooler(*this, framePointerRegister, destBufferGPR, temp1, temp2, fpTemp1, fpTemp2);

    RegisterAtOffsetList* allCalleeSaves = RegisterSet::vmCalleeSaveRegisterOffsets();
    const RegisterAtOffsetList* currentCalleeSaves = &RegisterAtOffsetList::llintBaselineCalleeSaveRegisters();
    auto dontCopyRegisters = RegisterSet::stackRegisters();
    unsigned registerCount = allCalleeSaves->registerCount();

    unsigned i = 0;
    for (; i < registerCount; i++) {
        RegisterAtOffset entry = allCalleeSaves->at(i);
        if (dontCopyRegisters.contains(entry.reg(), IgnoreVectors))
            continue;
        RegisterAtOffset* currentFrameEntry = currentCalleeSaves->find(entry.reg());

        if (!entry.reg().isGPR())
            break;
        if (currentFrameEntry)
            spooler.loadGPR(currentFrameEntry->offset());
        else
            spooler.copyGPR(entry.reg().gpr());
        spooler.storeGPR(entry.offset());
    }
    spooler.finalizeGPR();

    for (; i < registerCount; i++) {
        RegisterAtOffset entry = allCalleeSaves->at(i);
        if (dontCopyRegisters.contains(entry.reg(), IgnoreVectors))
            continue;
        RegisterAtOffset* currentFrameEntry = currentCalleeSaves->find(entry.reg());

        RELEASE_ASSERT(entry.reg().isFPR());
        if (currentFrameEntry)
            spooler.loadFPR(currentFrameEntry->offset());
        else
            spooler.copyFPR(entry.reg().fpr());
        spooler.storeFPR(entry.offset());
    }
    spooler.finalizeFPR();

#else
    UNUSED_PARAM(topEntryFrame);
    UNUSED_PARAM(usedRegisters);
#endif
}

void AssemblyHelpers::emitSaveOrCopyLLIntBaselineCalleeSavesFor(CodeBlock* codeBlock, VirtualRegister offsetVirtualRegister, RestoreTagRegisterMode tagRegisterMode, GPRReg temp1, GPRReg temp2, GPRReg temp3)
{
    ASSERT_UNUSED(codeBlock, codeBlock);
    ASSERT(JITCode::isBaselineCode(codeBlock->jitType()));
    ASSERT(codeBlock->jitCode()->calleeSaveRegisters() == &RegisterAtOffsetList::llintBaselineCalleeSaveRegisters());

    const RegisterAtOffsetList* calleeSaves = &RegisterAtOffsetList::llintBaselineCalleeSaveRegisters();
    auto dontSaveRegisters = RegisterSet::stackRegisters();
    unsigned registerCount = calleeSaves->registerCount();

    GPRReg dstBufferGPR = temp1;
    addPtr(TrustedImm32(offsetVirtualRegister.offsetInBytes()), framePointerRegister, dstBufferGPR);

    CopySpooler spooler(*this, framePointerRegister, dstBufferGPR, temp2, temp3);

    for (unsigned i = 0; i < registerCount; i++) {
        RegisterAtOffset entry = calleeSaves->at(i);
        if (dontSaveRegisters.contains(entry.reg(), IgnoreVectors))
            continue;
        RELEASE_ASSERT(entry.reg().isGPR());

#if USE(JSVALUE32_64)
        UNUSED_PARAM(tagRegisterMode);
#else
        if (tagRegisterMode == CopyBaselineCalleeSavedRegistersFromBaseFrame)
            spooler.loadGPR(entry.offset());
        else
#endif
            spooler.copyGPR(entry.reg().gpr());
        spooler.storeGPR(entry.offset());
    }
    spooler.finalizeGPR();
}

void AssemblyHelpers::getArityPadding(VM& vm, unsigned numberOfParameters, GPRReg argumentCountIncludingThisGPR, GPRReg paddingOutputGPR, GPRReg scratchGPR0, GPRReg scratchGPR1, JumpList& stackOverflow)
{
    ASSERT(argumentCountIncludingThisGPR != paddingOutputGPR);
    ASSERT(numberOfParameters >= 1);

    // padding is `align2(numParameters + CallFrame::headerSizeInRegisters) - (argumentCountIncludingThis + CallFrame::headerSizeInRegisters)`
    //
    // 1. If `align2(numParameters + CallFrame::headerSizeInRegisters)` == argumentCountIncludingThis + CallFrame::headerSizeInRegisters, then
    //    padding is `numParameters + CallFrame::headerSizeInRegisters - argumentCountIncludingThis - CallFrame::headerSizeInRegisters` == `numParameters - argumentCountIncludingThis`.
    //    Since we already checked `numParameters > argumentCountIncludingThis`, it will be 1~.
    // 2. If `align2(numParameters + CallFrame::headerSizeInRegisters)` == argumentCountIncludingThis + CallFrame::headerSizeInRegisters + 1, then
    //    padding is `numParameters + CallFrame::headerSizeInRegisters + 1 - argumentCountIncludingThis - CallFrame::headerSizeInRegisters` == `numParameters + 1 - argumentCountIncludingThis`.
    //    Since we already checked `numParameters > argumentCountIncludingThis`, it will be 2~.
    //
    //  Then, we return align2(padding).
    static_assert(stackAlignmentRegisters() == 2);
    if (WTF::roundUpToMultipleOf(stackAlignmentRegisters(), numberOfParameters + CallFrame::headerSizeInRegisters) == numberOfParameters + CallFrame::headerSizeInRegisters)
        move(TrustedImm32(numberOfParameters), paddingOutputGPR);
    else
        move(TrustedImm32(numberOfParameters + 1), paddingOutputGPR);
    sub32(paddingOutputGPR, argumentCountIncludingThisGPR, paddingOutputGPR);

    add32(TrustedImm32(1), paddingOutputGPR, scratchGPR0);
    and32(TrustedImm32(~1U), scratchGPR0);
    lshiftPtr(TrustedImm32(3), scratchGPR0);
    subPtr(stackPointerRegister, scratchGPR0, scratchGPR1);
    stackOverflow.append(branchPtrAgainstSoftStackLimit(vm, Above, scratchGPR1)); // UNGIL §A.2.2 (AB-17): per-lite GIL-off; unsigned, matching the landed AbsoluteAddress form.
}

#if USE(JSVALUE64)

AssemblyHelpers::JumpList AssemblyHelpers::branchIfResizableOrGrowableSharedTypedArrayIsOutOfBounds(GPRReg baseGPR, GPRReg scratchGPR, GPRReg scratch2GPR, std::optional<TypedArrayType> typedArrayType)
{
    ASSERT(scratchGPR != scratch2GPR);
    // valueGPR can be baseGPR

    JumpList outOfBounds;

    outOfBounds.append(branchTestPtr(MacroAssembler::Zero, MacroAssembler::Address(baseGPR, JSArrayBufferView::offsetOfVector())));

    load8(Address(baseGPR, JSArrayBufferView::offsetOfMode()), scratchGPR);
    and32(TrustedImm32(resizabilityAndAutoLengthMask), scratchGPR, scratch2GPR);
    auto canUseRawFieldsDirectly = branch32(BelowOrEqual, scratch2GPR, TrustedImm32(isGrowableSharedMode));

    // Three cases come here.
    //     GrowableSharedAutoLengthWastefulTypedArray
    //     ResizableNonSharedWastefulTypedArray
    //     ResizableNonSharedAutoLengthWastefulTypedArray

    // r48: segment-aware butterfly -> arrayBuffer load (clobbers scratchGPR
    // flag-on; mode is re-loaded for the isGrowableShared test below).
    AssemblyHelpersInternal::emitLoadTypedArrayArrayBuffer(*this, baseGPR, scratch2GPR, scratchGPR);
#if USE(JSVALUE64)
    if (Options::useJSThreads()) [[unlikely]]
        load8(Address(baseGPR, JSArrayBufferView::offsetOfMode()), scratchGPR);
#endif

    auto isGrowableShared = branchTest32(NonZero, scratchGPR, TrustedImm32(isGrowableSharedMode));
#if USE(LARGE_TYPED_ARRAYS)
    load64(Address(scratch2GPR, ArrayBuffer::offsetOfSizeInBytes()), scratch2GPR);
#else
    load32(Address(scratch2GPR, ArrayBuffer::offsetOfSizeInBytes()), scratch2GPR);
#endif
    auto loadedFromResizable = jump();

    isGrowableShared.link(this);
    loadPtr(Address(scratch2GPR, ArrayBuffer::offsetOfShared()), scratch2GPR);
#if USE(LARGE_TYPED_ARRAYS)
    atomicLoad64(Address(scratch2GPR, SharedArrayBufferContents::offsetOfSizeInBytes()), scratch2GPR);
#else
    atomicLoad32(Address(scratch2GPR, SharedArrayBufferContents::offsetOfSizeInBytes()), scratch2GPR);
#endif

    loadedFromResizable.link(this);

    // It never overflows since it is already checked at the construction.
#if USE(LARGE_TYPED_ARRAYS)
    if (typedArrayType) {
        load64(Address(baseGPR, JSArrayBufferView::offsetOfLength()), scratchGPR);
        if (elementSize(typedArrayType.value()) > 1)
            lshift64(TrustedImm32(logElementSize(typedArrayType.value())), scratchGPR);
    } else {
        load8(Address(baseGPR, JSObject::typeInfoTypeOffset()), scratchGPR);
        addPtr(TrustedImmPtr(logElementSizes), scratchGPR);
        load8(Address(scratchGPR, -FirstTypedArrayType), scratchGPR);
        lshift64(Address(baseGPR, JSArrayBufferView::offsetOfLength()), scratchGPR, scratchGPR);
    }
    add64(Address(baseGPR, JSArrayBufferView::offsetOfByteOffset()), scratchGPR);
    outOfBounds.append(branch64(Above, scratchGPR, scratch2GPR));
#else
    if (typedArrayType) {
        load32(Address(baseGPR, JSArrayBufferView::offsetOfLength()), scratchGPR);
        if (elementSize(typedArrayType.value()) > 1)
            lshift32(TrustedImm32(logElementSize(typedArrayType.value())), scratchGPR);
    } else {
        load8(Address(baseGPR, JSObject::typeInfoTypeOffset()), scratchGPR);
        addPtr(TrustedImmPtr(logElementSizes), scratchGPR);
        load8(Address(scratchGPR, -FirstTypedArrayType), scratchGPR);
        lshift32(Address(baseGPR, JSArrayBufferView::offsetOfLength()), scratchGPR, scratchGPR);
    }
    add32(Address(baseGPR, JSArrayBufferView::offsetOfByteOffset()), scratchGPR);
    outOfBounds.append(branch32(Above, scratchGPR, scratch2GPR));
#endif

    canUseRawFieldsDirectly.link(this);

    return outOfBounds;
}

std::tuple<AssemblyHelpers::Jump, AssemblyHelpers::JumpList> AssemblyHelpers::loadTypedArrayByteLengthImpl(GPRReg baseGPR, GPRReg valueGPR, GPRReg scratchGPR, GPRReg scratch2GPR, std::optional<TypedArrayType> typedArrayType, TypedArrayField field)
{
    ASSERT(scratchGPR != scratch2GPR);
    ASSERT(scratchGPR != valueGPR);
    // scratch2GPR can be valueGPR
    // valueGPR can be baseGPR

    load8(Address(baseGPR, JSArrayBufferView::offsetOfMode()), scratchGPR);
    and32(TrustedImm32(resizabilityAndAutoLengthMask), scratchGPR, scratch2GPR);
    auto canUseRawFieldsDirectly = branch32(BelowOrEqual, scratch2GPR, TrustedImm32(isGrowableSharedMode));

    // Three cases come here.
    //     GrowableSharedAutoLengthWastefulTypedArray
    //     ResizableNonSharedWastefulTypedArray
    //     ResizableNonSharedAutoLengthWastefulTypedArray

    if (typedArrayType && typedArrayType.value() == TypeDataView)
        loadPtr(Address(baseGPR, JSDataView::offsetOfBuffer()), scratch2GPR);
    else {
        // r48: segment-aware butterfly -> arrayBuffer load (clobbers
        // scratchGPR flag-on; mode is re-loaded for the isGrowableShared test
        // below).
        AssemblyHelpersInternal::emitLoadTypedArrayArrayBuffer(*this, baseGPR, scratch2GPR, scratchGPR);
#if USE(JSVALUE64)
        if (Options::useJSThreads()) [[unlikely]]
            load8(Address(baseGPR, JSArrayBufferView::offsetOfMode()), scratchGPR);
#endif
    }

    auto isGrowableShared = branchTest32(NonZero, scratchGPR, TrustedImm32(isGrowableSharedMode));
#if USE(LARGE_TYPED_ARRAYS)
    load64(Address(scratch2GPR, ArrayBuffer::offsetOfSizeInBytes()), scratch2GPR);
#else
    load32(Address(scratch2GPR, ArrayBuffer::offsetOfSizeInBytes()), scratch2GPR);
#endif
    auto loadedFromResizable = jump();

    isGrowableShared.link(this);
    loadPtr(Address(scratch2GPR, ArrayBuffer::offsetOfShared()), scratch2GPR);
#if USE(LARGE_TYPED_ARRAYS)
    atomicLoad64(Address(scratch2GPR, SharedArrayBufferContents::offsetOfSizeInBytes()), scratch2GPR);
#else
    atomicLoad32(Address(scratch2GPR, SharedArrayBufferContents::offsetOfSizeInBytes()), scratch2GPR);
#endif

    loadedFromResizable.link(this);

    // It never overflows since it is already checked at the construction.
#if USE(LARGE_TYPED_ARRAYS)
    if (typedArrayType) {
        load64(Address(baseGPR, JSArrayBufferView::offsetOfLength()), scratchGPR);
        if (elementSize(typedArrayType.value()) > 1)
            lshift64(TrustedImm32(logElementSize(typedArrayType.value())), scratchGPR);
    } else {
        load8(Address(baseGPR, JSObject::typeInfoTypeOffset()), scratchGPR);
        addPtr(TrustedImmPtr(logElementSizes), scratchGPR);
        load8(Address(scratchGPR, -FirstTypedArrayType), scratchGPR);
        lshift64(Address(baseGPR, JSArrayBufferView::offsetOfLength()), scratchGPR, scratchGPR);
    }
    add64(Address(baseGPR, JSArrayBufferView::offsetOfByteOffset()), scratchGPR);
    auto outOfBounds = branch64(Above, scratchGPR, scratch2GPR);
#else
    if (typedArrayType) {
        load32(Address(baseGPR, JSArrayBufferView::offsetOfLength()), scratchGPR);
        if (elementSize(typedArrayType.value()) > 1)
            lshift32(TrustedImm32(logElementSize(typedArrayType.value())), scratchGPR);
    } else {
        load8(Address(baseGPR, JSObject::typeInfoTypeOffset()), scratchGPR);
        addPtr(TrustedImmPtr(logElementSizes), scratchGPR);
        load8(Address(scratchGPR, -FirstTypedArrayType), scratchGPR);
        lshift32(Address(baseGPR, JSArrayBufferView::offsetOfLength()), scratchGPR, scratchGPR);
    }
    add32(Address(baseGPR, JSArrayBufferView::offsetOfByteOffset()), scratchGPR);
    auto outOfBounds = branch32(Above, scratchGPR, scratch2GPR);
#endif

    auto nonAutoLength = branchTest8(Zero, Address(baseGPR, JSArrayBufferView::offsetOfMode()), TrustedImm32(isAutoLengthMode));

    JumpList doneCases;

#if USE(LARGE_TYPED_ARRAYS)
    if (field == TypedArrayField::ByteLength) {
        sub64(scratch2GPR, scratchGPR, valueGPR);
        ASSERT(typedArrayType);
        if (elementSize(typedArrayType.value()) > 1)
            and64(TrustedImm64(~static_cast<uint64_t>(elementSize(typedArrayType.value()) - 1)), valueGPR);
    } else {
        if (typedArrayType) {
            sub64(scratch2GPR, scratchGPR, valueGPR);
            if (elementSize(typedArrayType.value()) > 1)
                urshift64(TrustedImm32(logElementSize(typedArrayType.value())), valueGPR);
        } else {
            sub64(scratch2GPR, scratchGPR, scratch2GPR); // Do not change valueGPR until we use baseGPR.
            load8(Address(baseGPR, JSObject::typeInfoTypeOffset()), scratchGPR);
            addPtr(TrustedImmPtr(logElementSizes), scratchGPR);
            load8(Address(scratchGPR, -FirstTypedArrayType), scratchGPR);
            urshift64(scratch2GPR, scratchGPR, valueGPR);
        }
    }
#else
    if (field == TypedArrayField::ByteLength) {
        sub32(scratch2GPR, scratchGPR, valueGPR);
        ASSERT(typedArrayType);
        if (elementSize(typedArrayType.value()) > 1)
            and32(TrustedImm32(~static_cast<uint32_t>(elementSize(typedArrayType.value()) - 1)), valueGPR);
    } else {
        if (typedArrayType) {
            sub32(scratch2GPR, scratchGPR, valueGPR);
            if (elementSize(typedArrayType.value()) > 1)
                urshift32(TrustedImm32(logElementSize(typedArrayType.value())), valueGPR);
        } else {
            sub32(scratch2GPR, scratchGPR, scratch2GPR); // Do not change valueGPR until we use baseGPR.
            load8(Address(baseGPR, JSObject::typeInfoTypeOffset()), scratchGPR);
            addPtr(TrustedImmPtr(logElementSizes), scratchGPR);
            load8(Address(scratchGPR, -FirstTypedArrayType), scratchGPR);
            urshift32(scratch2GPR, scratchGPR, valueGPR);
        }
    }
#endif
    doneCases.append(jump());

    nonAutoLength.link(this);
    canUseRawFieldsDirectly.link(this);
#if USE(LARGE_TYPED_ARRAYS)
    load64(MacroAssembler::Address(baseGPR, JSArrayBufferView::offsetOfLength()), valueGPR);
    if (field == TypedArrayField::ByteLength) {
        ASSERT(typedArrayType);
        if (elementSize(typedArrayType.value()) > 1)
            lshift64(TrustedImm32(logElementSize(typedArrayType.value())), valueGPR);
    }
#else
    load32(MacroAssembler::Address(baseGPR, JSArrayBufferView::offsetOfLength()), valueGPR);
    if (field == TypedArrayField::ByteLength) {
        ASSERT(typedArrayType);
        if (elementSize(typedArrayType.value()) > 1)
            lshift32(TrustedImm32(logElementSize(typedArrayType.value())), valueGPR);
    }
#endif
    doneCases.append(jump());

    return { outOfBounds, doneCases };
}

void AssemblyHelpers::loadTypedArrayByteLengthCommonImpl(GPRReg baseGPR, GPRReg valueGPR, GPRReg scratchGPR, GPRReg scratch2GPR, std::optional<TypedArrayType> typedArrayType, TypedArrayField field)
{
    auto [outOfBounds, doneCases] = loadTypedArrayByteLengthImpl(baseGPR, valueGPR, scratchGPR, scratch2GPR, typedArrayType, field);
    outOfBounds.link(this);
    move(TrustedImm32(0), valueGPR);
    doneCases.link(this);
}

void AssemblyHelpers::loadTypedArrayByteLength(GPRReg baseGPR, GPRReg valueGPR, GPRReg scratchGPR, GPRReg scratch2GPR, TypedArrayType typedArrayType)
{
    loadTypedArrayByteLengthCommonImpl(baseGPR, valueGPR, scratchGPR, scratch2GPR, typedArrayType, TypedArrayField::ByteLength);
}

std::tuple<AssemblyHelpers::Jump, AssemblyHelpers::JumpList> AssemblyHelpers::loadDataViewByteLength(GPRReg baseGPR, GPRReg valueGPR, GPRReg scratchGPR, GPRReg scratch2GPR, TypedArrayType typedArrayType)
{
    return loadTypedArrayByteLengthImpl(baseGPR, valueGPR, scratchGPR, scratch2GPR, typedArrayType, TypedArrayField::ByteLength);
}

void AssemblyHelpers::loadTypedArrayLength(GPRReg baseGPR, GPRReg valueGPR, GPRReg scratchGPR, GPRReg scratch2GPR, std::optional<TypedArrayType> typedArrayType)
{
    loadTypedArrayByteLengthCommonImpl(baseGPR, valueGPR, scratchGPR, scratch2GPR, typedArrayType, TypedArrayField::Length);
}

#endif // ENABLE(JSVALUE64)

#if ENABLE(WEBASSEMBLY)
#if CPU(ARM64) || CPU(X86_64) || CPU(RISCV64) || CPU(ARM)
AssemblyHelpers::JumpList AssemblyHelpers::checkWasmStackOverflow(GPRReg instanceGPR, TrustedImm32 checkSize, GPRReg framePointerGPR)
{
#if CPU(ARM64)
    loadPtr(Address(instanceGPR, JSWebAssemblyInstance::offsetOfSoftStackLimit()), getCachedMemoryTempRegisterIDAndInvalidate());
    JumpList overflow;
    // Because address is within 48bit, this addition never causes overflow.
    addPtr(checkSize, memoryTempRegister); // TrustedImm32 would use dataTempRegister. Thus let's have limit in memoryTempRegister.
    overflow.append(branchPtr(LessThan, framePointerGPR, memoryTempRegister));
    return overflow;
#elif CPU(X86_64) || CPU(ARM)
    loadPtr(Address(instanceGPR, JSWebAssemblyInstance::offsetOfSoftStackLimit()), scratchRegister());
    JumpList overflow;
    // Because address is within 48bit, this addition never causes overflow.
    addPtr(checkSize, scratchRegister());
    overflow.append(branchPtr(LessThan, framePointerGPR, scratchRegister()));
    return overflow;
#elif CPU(RISCV64)
    loadPtr(Address(instanceGPR, JSWebAssemblyInstance::offsetOfSoftStackLimit()), memoryTempRegister);
    JumpList overflow;
    // Because address is within 48bit, this addition never causes overflow.
    addPtr(checkSize, memoryTempRegister); // TrustedImm32 would use dataTempRegister. Thus let's have limit in memoryTempRegister.
    overflow.append(branchPtr(LessThan, framePointerGPR, memoryTempRegister));
    return overflow;
#endif
}
#endif
#endif // ENABLE(WEBASSEMBLY)

} // namespace JSC

#endif // ENABLE(JIT)

WTF_ALLOW_UNSAFE_BUFFER_USAGE_END