FTLThunks.cpp

/*
 * Copyright (C) 2013-2023 Apple Inc. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
 */

#include "config.h"
#include "FTLThunks.h"

#if ENABLE(FTL_JIT)

#include "AssemblyHelpersSpoolers.h"
#include "DFGOSRExitCompilerCommon.h"
#include "FTLJITCode.h"
#include "FTLLazySlowPath.h"
#include "FTLOSRExitCompiler.h"
#include "FTLOperations.h"
#include "FTLSaveRestore.h"
#include "GPRInfo.h"
#include "LinkBuffer.h"
#include <wtf/TZoneMallocInlines.h>

namespace JSC { namespace FTL {

WTF_MAKE_TZONE_ALLOCATED_IMPL(Thunks);

using namespace DFG;

enum class FrameAndStackAdjustmentRequirement {
    Needed, 
    NotNeeded 
};

static MacroAssemblerCodeRef<JITThunkPtrTag> genericGenerationThunkGenerator(
    VM& vm, CodePtr<CFunctionPtrTag> generationFunction, PtrTag resultTag, const char* name, unsigned extraPopsToRestore, FrameAndStackAdjustmentRequirement frameAndStackAdjustmentRequirement,
    void (*thinPrefix)(AssemblyHelpers&) = nullptr)
{
    AssemblyHelpers jit(nullptr);

    // SCALEBENCH §42 thin-thunk: gilOff-only fast prefix that bypasses the
    // saveAllRegisters / operation-call / restoreAllRegisters body when the
    // steady-state answer is already published (lazy slow path's
    // m_stubCodePtr). nullptr for OSR-exit and for GIL-on, so flag-off /
    // GIL-on emit the IDENTICAL byte sequence below (no prefix → first
    // emitted instruction is the pushToSave(framePointerRegister) that
    // upstream emits today).
    if (thinPrefix) [[unlikely]]
        thinPrefix(jit);

    if (frameAndStackAdjustmentRequirement == FrameAndStackAdjustmentRequirement::Needed) {
        // This needs to happen before we use the scratch buffer because this function also uses the scratch buffer.
        adjustFrameAndStackInOSRExitCompilerThunk<FTL::JITCode>(jit, vm, JITType::FTLJIT);
    }
    
    // Note that the "return address" will be the ID that we pass to the generation function.

    constexpr GPRReg stackPointerRegister = MacroAssembler::stackPointerRegister;
    constexpr GPRReg framePointerRegister = MacroAssembler::framePointerRegister;
    constexpr ptrdiff_t pushToSaveByteOffset = MacroAssembler::pushToSaveByteOffset();
    ptrdiff_t stackMisalignment = pushToSaveByteOffset;

    // Pretend that we're a C call frame.
    jit.pushToSave(framePointerRegister);
    jit.move(stackPointerRegister, framePointerRegister);
    stackMisalignment += pushToSaveByteOffset;

    // Now create ourselves enough stack space to give saveAllRegisters() a scratch slot.
    unsigned numberOfRequiredPops = 0;
    do {
        stackMisalignment += pushToSaveByteOffset;
        numberOfRequiredPops++;
    } while (stackMisalignment % stackAlignmentBytes());
    jit.subPtr(MacroAssembler::TrustedImm32(numberOfRequiredPops * pushToSaveByteOffset), stackPointerRegister);

    // UNGIL §A.1.6 (ANNEX A16, U-T4b): the generation thunk is shared by all
    // threads of this VM. gilOff, two threads firing exits concurrently would
    // clobber each other's full register dump in a baked buffer, so the
    // gilOff-mode thunk bakes a process-wide ScratchBufferRegistry INDEX and
    // resolves the CURRENT lite's buffer per use (loadVMLite -> segment ->
    // [index]; rematerialized per §A.1.2). GIL-on/flag-off keeps the baked
    // address byte-for-byte.
    const bool bakedIndexMode = vm.gilOff();
    unsigned bakedIndex = std::numeric_limits<unsigned>::max();
    char* buffer = nullptr;
    if (bakedIndexMode) [[unlikely]]
        bakedIndex = vm.allocateBakedScratchBufferIndex(requiredScratchMemorySizeInBytes());
    else {
        ScratchBuffer* scratchBuffer = vm.scratchBufferForSize(requiredScratchMemorySizeInBytes());
        buffer = static_cast<char*>(scratchBuffer->dataBuffer());
    }
    auto materializeBufferBase = scopedLambda<void(AssemblyHelpers&, GPRReg)>(
        [&] (AssemblyHelpers& jit, GPRReg dest) {
            materializeBakedScratchBufferDataPointer(jit, bakedIndex, dest);
        });

    if (bakedIndexMode) [[unlikely]]
        saveAllRegisters(jit, materializeBufferBase);
    else
        saveAllRegisters(jit, buffer);

    jit.loadPtr(CCallHelpers::Address(framePointerRegister), GPRInfo::argumentGPR0);
    jit.peek(
        GPRInfo::argumentGPR1,
        (stackMisalignment - pushToSaveByteOffset) / sizeof(void*));
    jit.prepareCallOperation(vm);
    jit.callOperation<OperationPtrTag>(generationFunction.retagged<OperationPtrTag>());

    // At this point we want to make a tail call to what was returned to us in the
    // returnValueGPR. But at the same time as we do this, we must restore all registers.
    // The way we will accomplish this is by arranging to have the tail call target in the
    // return address "slot" (be it a register or the stack).
    
    jit.move(GPRInfo::returnValueGPR, GPRInfo::regT0);

    // Prepare for tail call.

    jit.loadPtr(MacroAssembler::Address(stackPointerRegister, numberOfRequiredPops * pushToSaveByteOffset), framePointerRegister);

    // When we came in here, there was an additional thing pushed to the stack (extraPopsToRestore).
    // Some clients want it popped before proceeding. Also add 1 for the pushToSave of the framePointerRegister.
    numberOfRequiredPops += 1 + extraPopsToRestore;
    jit.addPtr(MacroAssembler::TrustedImm32(numberOfRequiredPops * pushToSaveByteOffset), stackPointerRegister);

    // Put the return address wherever the return instruction wants it. On all platforms, this
    // ensures that the return address is out of the way of register restoration.
    jit.restoreReturnAddressBeforeReturn(GPRInfo::regT0);

#if CPU(ARM64E)
    jit.untagPtr(resultTag, AssemblyHelpers::linkRegister);
    jit.validateUntaggedPtr(AssemblyHelpers::linkRegister);
    jit.tagReturnAddress();
#else
    UNUSED_PARAM(resultTag);
#endif

    if (bakedIndexMode) [[unlikely]]
        restoreAllRegisters(jit, materializeBufferBase);
    else
        restoreAllRegisters(jit, buffer);

    jit.ret();
    
    LinkBuffer patchBuffer(jit, GLOBAL_THUNK_ID, LinkBuffer::Profile::FTLThunk);
    return FINALIZE_THUNK(patchBuffer, JITThunkPtrTag, nullptr, "%s", name);
}

MacroAssemblerCodeRef<JITThunkPtrTag> osrExitGenerationThunkGenerator(VM& vm)
{
    unsigned extraPopsToRestore = 0;
    return genericGenerationThunkGenerator(
        vm, operationCompileFTLOSRExit, OSRExitPtrTag, "FTL OSR exit generation thunk", extraPopsToRestore, FrameAndStackAdjustmentRequirement::Needed);
}

// SCALEBENCH §42 thin-thunk (GILOFF-TAX-EVIDENCE.md §#1, residual #2):
// gilOff-dedicated thin prefix for the FTL lazy-slow-path steady state.
//
// Rationale: gilOff (UNGIL U-T4b, FTLLazySlowPath.cpp:73-87) leaves the
// patchable jump UNPATCHED, so EVERY traversal — 36.4 M of them on intcs W=1
// after the §42 TLC-slot fix took CompleteSubspace allocations off this path
// — runs the full genericGenerationThunkGenerator body: saveAllRegisters
// (full scalar dump to a per-lite scratch buffer), a real C call into
// operationCompileFTLLazySlowPath, restoreAllRegisters, ret. perf attributes
// ~910 ms of intcs W=1 self time (~25 ns/traversal) to that one ~1.5 KB JIT
// range; the operation body's only steady-state work is the T8
// stubCodePtrConcurrently() acquire-load (FTLOperations.cpp:987).
//
// This prefix replays JUST that acquire-load in JIT code: callFrame →
// codeBlock → m_jitCode (ConcurrentJITCodePtr; one raw word, JITCode.h:428)
// → FTL::JITCode::lazySlowPaths[index] → m_stubCodePtr. Non-null (steady
// state after first compile) → restore the two spilled scratches, pop the
// late-path's index push (the same push the full body's extraPopsToRestore=1
// removes), tail-jump to the stub. Null → restore both scratches and FALL
// THROUGH to today's full body with sp at exactly its on-entry layout, so
// first-compile / race-loser semantics are byte-for-byte the existing
// double-checked-publication path under ftlLazySlowPathGenerationLock.
//
// Register discipline: at thunk entry every FTL-live register is live EXCEPT
// macroClobberedGPRs, which the lazySlowPath() patchpoint explicitly clobbers
// (FTLLowerDFGToB3.cpp:25544 result->clobber(RegisterSet::macroClobberedGPRs())).
// We spill exactly two ordinary GPRs for the load chain and use
// GPRInfo::patchpointScratchRegister (= the one register the patchpoint
// contract guarantees is dead: r11 / ip0 / x30) only as the final jump
// target, after both spills are restored — the only instructions between its
// load and the farJump are popToRestore (raw post-index ldr / pop) which
// touch no macro temp.
//
// Orthogonal to the §42 iso-TLC-slot fix: that removes the dominant MakeRope
// traversals; this cheapens whatever lazy-slow-path traversals REMAIN
// (write-barrier slow paths, iso-subspace allocations until iso-TLC lands,
// every other lazySlowPath() consumer in FTLLowerDFGToB3.cpp). gilOff-gated:
// GIL-on never reaches this function (lazySlowPathGenerationThunkGenerator
// passes nullptr) so the GIL-on / flag-off thunk is byte-identical.
static void emitLazySlowPathThinPrefix(AssemblyHelpers& jit)
{
    using MA = AssemblyHelpers;
    constexpr GPRReg scratch0 = GPRInfo::regT2;
    constexpr GPRReg scratch1 = GPRInfo::regT3;
    constexpr GPRReg targetGPR = GPRInfo::patchpointScratchRegister;
    static_assert(scratch0 != scratch1 && scratch0 != targetGPR && scratch1 != targetGPR);
    static_assert(scratch0 != GPRInfo::callFrameRegister && scratch1 != GPRInfo::callFrameRegister);
    constexpr ptrdiff_t pushToSaveByteOffset = MacroAssembler::pushToSaveByteOffset();

    // On entry: [sp + 0] = the late path's pushToSaveImmediateWithoutTouchingRegisters(index)
    // (FTLLowerDFGToB3.cpp:25572). cfr = the executing FTL frame = callFrame.
    jit.pushToSave(scratch0);
    jit.pushToSave(scratch1);
    // [sp + 0]              = scratch1 saved
    // [sp + pushToSaveByteOffset]   = scratch0 saved
    // [sp + 2*pushToSaveByteOffset] = index (low 32 bits)

    // index → scratch0
    jit.load32(MA::Address(MacroAssembler::stackPointerRegister, 2 * pushToSaveByteOffset), scratch0);
    // codeBlock → scratch1
    jit.loadPtr(MA::Address(GPRInfo::callFrameRegister, static_cast<int>(CallFrameSlot::codeBlock) * static_cast<int>(sizeof(Register))), scratch1);
    // FTL::JITCode* → scratch1 (ConcurrentJITCodePtr is one JITCode* word at
    // jitCodeOffset(); FTL::JITCode is single-inheritance from JSC::JITCode so
    // the base pointer is the derived pointer — same identity
    // operationCompileFTLLazySlowPath relies on via jitCodeRawPtr()->ftl()).
    static_assert(sizeof(ConcurrentJITCodePtr) == sizeof(JSC::JITCode*));
    jit.loadPtr(MA::Address(scratch1, CodeBlock::jitCodeOffset()), scratch1);
    // lazySlowPaths.data() → scratch1
    static_assert(sizeof(std::unique_ptr<LazySlowPath>) == sizeof(LazySlowPath*));
    constexpr ptrdiff_t lazySlowPathsBufferOffset = OBJECT_OFFSETOF(JITCode, lazySlowPaths) + Vector<std::unique_ptr<LazySlowPath>>::dataMemoryOffset();
    jit.loadPtr(MA::Address(scratch1, lazySlowPathsBufferOffset), scratch1);
    // lazySlowPaths[index] (LazySlowPath*) → scratch1
    jit.loadPtr(MA::BaseIndex(scratch1, scratch0, MA::ScalePtr), scratch1);
    // m_stubCodePtr (acquire) → scratch1. x86_64 TSO: plain load IS acquire.
    // ARM64: ldar pairs with generate()'s release-store so a non-null read
    // here observes a fully-constructed stub (data side; i-cache coherence is
    // the existing T8 contract — same as the C++ acquire-load fast path this
    // mirrors, FTLOperations.cpp:987).
#if CPU(ARM64)
    jit.loadAcq64(MA::Address(scratch1, LazySlowPath::offsetOfStubCodePtr()), scratch1);
#else
    jit.loadPtr(MA::Address(scratch1, LazySlowPath::offsetOfStubCodePtr()), scratch1);
#endif
    auto needGenerate = jit.branchTestPtr(MA::Zero, scratch1);

    // Steady state: tail-call the already-generated stub. Park the target in
    // the patchpoint-dead scratch, restore both spills and the index push so
    // sp is exactly what a directly-repatched jump would have seen, then go.
    jit.move(scratch1, targetGPR);
    jit.popToRestore(scratch1);
    jit.popToRestore(scratch0);
    jit.addPtr(MA::TrustedImm32(pushToSaveByteOffset), MacroAssembler::stackPointerRegister); // drop index push (= extraPopsToRestore=1).
    jit.farJump(targetGPR, JITStubRoutinePtrTag);

    // First compile (or publication race): restore and fall through to the
    // full saveAllRegisters / operationCompileFTLLazySlowPath body with sp at
    // its on-entry [index] layout.
    needGenerate.link(&jit);
    jit.popToRestore(scratch1);
    jit.popToRestore(scratch0);
}

MacroAssemblerCodeRef<JITThunkPtrTag> lazySlowPathGenerationThunkGenerator(VM& vm)
{
    unsigned extraPopsToRestore = 1;
    // SCALEBENCH §42 thin-thunk: gilOff-only thin prefix; GIL-on / flag-off
    // pass no prefix and emit the byte-identical upstream thunk.
    return genericGenerationThunkGenerator(
        vm, operationCompileFTLLazySlowPath, JITStubRoutinePtrTag, "FTL lazy slow path generation thunk", extraPopsToRestore, FrameAndStackAdjustmentRequirement::NotNeeded,
        vm.gilOff() ? emitLazySlowPathThinPrefix : nullptr);
}

static void registerClobberCheck(AssemblyHelpers& jit, RegisterSet dontClobber)
{
    ASSERT(Options::clobberAllRegsInFTLICSlowPath());
    RegisterSet clobber = RegisterSet::registersToSaveForJSCall(RegisterSet::allScalarRegisters());
    clobber.exclude(dontClobber);
    auto wholeClobberedRegisters = clobber.normalizeWidths();
    
    GPRReg someGPR = InvalidGPRReg;
    for (Reg reg = Reg::first(); reg <= Reg::last(); reg = reg.next()) {
        if (!wholeClobberedRegisters.contains(reg, IgnoreVectors) || !reg.isGPR())
            continue;
        
        jit.move(AssemblyHelpers::TrustedImm32(0x1337beef), reg.gpr());
        someGPR = reg.gpr();
    }
    
    for (Reg reg = Reg::first(); reg <= Reg::last(); reg = reg.next()) {
        if (!wholeClobberedRegisters.contains(reg, IgnoreVectors) || !reg.isFPR())
            continue;
        
        jit.move64ToDouble(someGPR, reg.fpr());
    }
}

MacroAssemblerCodeRef<JITThunkPtrTag> slowPathCallThunkGenerator(VM& vm, const SlowPathCallKey& key)
{
    AssemblyHelpers jit(nullptr);
    jit.tagReturnAddress();

    // We want to save the given registers at the given offset, then we want to save the
    // old return address somewhere past that offset, and then finally we want to make the
    // call.
    
    size_t currentOffset = key.offset() + sizeof(void*);
    
#if CPU(X86_64)
    currentOffset += sizeof(void*);
#endif

    AssemblyHelpers::StoreRegSpooler storeSpooler(jit, MacroAssembler::stackPointerRegister);

    for (MacroAssembler::RegisterID reg = MacroAssembler::firstRegister(); reg <= MacroAssembler::lastRegister(); reg = static_cast<MacroAssembler::RegisterID>(reg + 1)) {
        if (!key.usedRegisters().contains(reg, IgnoreVectors))
            continue;
        storeSpooler.storeGPR({ reg, static_cast<ptrdiff_t>(currentOffset), conservativeWidthWithoutVectors(reg) });
        currentOffset += sizeof(void*);
    }
    storeSpooler.finalizeGPR();

    for (MacroAssembler::FPRegisterID reg = MacroAssembler::firstFPRegister(); reg <= MacroAssembler::lastFPRegister(); reg = static_cast<MacroAssembler::FPRegisterID>(reg + 1)) {
        if (!key.usedRegisters().contains(reg, IgnoreVectors))
            continue;
        storeSpooler.storeFPR({ reg, static_cast<ptrdiff_t>(currentOffset), conservativeWidthWithoutVectors(reg) });
        currentOffset += sizeof(double);
    }
    storeSpooler.finalizeFPR();

    jit.preserveReturnAddressAfterCall(GPRInfo::nonArgGPR1);
    jit.storePtr(GPRInfo::nonArgGPR1, AssemblyHelpers::Address(MacroAssembler::stackPointerRegister, key.offset()));
    jit.prepareCallOperation(vm);
    
    if (Options::clobberAllRegsInFTLICSlowPath()) [[unlikely]] {
        auto dontClobber = key.argumentRegistersIfClobberingCheckIsEnabled();
        if (!key.callTarget())
            dontClobber.add(GPRInfo::nonArgGPR0, IgnoreVectors);
        registerClobberCheck(jit, WTF::move(dontClobber));
    }

    AssemblyHelpers::Call call;
    if (key.callTarget())
        jit.callOperation<OperationPtrTag>(key.callTarget());
    else
        jit.call(CCallHelpers::Address(GPRInfo::nonArgGPR0, key.indirectOffset()), OperationPtrTag);

    jit.loadPtr(AssemblyHelpers::Address(MacroAssembler::stackPointerRegister, key.offset()), GPRInfo::nonPreservedNonReturnGPR);
    jit.restoreReturnAddressBeforeReturn(GPRInfo::nonPreservedNonReturnGPR);
    
    AssemblyHelpers::LoadRegSpooler loadSpooler(jit, MacroAssembler::stackPointerRegister);

    for (MacroAssembler::FPRegisterID reg = MacroAssembler::lastFPRegister(); ; reg = static_cast<MacroAssembler::FPRegisterID>(reg - 1)) {
        if (key.usedRegisters().contains(reg, IgnoreVectors)) {
            currentOffset -= sizeof(double);
            loadSpooler.loadFPR({ reg, static_cast<ptrdiff_t>(currentOffset), conservativeWidthWithoutVectors(reg) });
        }
        if (reg == MacroAssembler::firstFPRegister())
            break;
    }
    loadSpooler.finalizeFPR();

    for (MacroAssembler::RegisterID reg = MacroAssembler::lastRegister(); ; reg = static_cast<MacroAssembler::RegisterID>(reg - 1)) {
        if (key.usedRegisters().contains(reg, IgnoreVectors)) {
            currentOffset -= sizeof(void*);
            loadSpooler.loadGPR({ reg, static_cast<ptrdiff_t>(currentOffset), conservativeWidthWithoutVectors(reg) });
        }
        if (reg == MacroAssembler::firstRegister())
            break;
    }
    loadSpooler.finalizeGPR();

    jit.ret();

    LinkBuffer patchBuffer(jit, GLOBAL_THUNK_ID, LinkBuffer::Profile::FTLThunk);
    return FINALIZE_THUNK(patchBuffer, JITThunkPtrTag, nullptr, "FTL slow path call thunk for %s", toCString(key).data());
}

} } // namespace JSC::FTL

#endif // ENABLE(FTL_JIT)