Skip to content

Fix x64 float parameter bugs #2547

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
psychonic merged 3 commits into from
Aug 15, 2026
Merged

Fix x64 float parameter bugs #2547

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1988f48
Fix stack spill for float parameters
Mikusch Aug 15, 2026
9e17d7d
Fix invalid register in vararg float copy
Mikusch Aug 15, 2026
cc3de72
Fix disp32 encoding, REX operand order and stack offset truncation in…
psychonic Aug 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 17 additions & 14 deletions extensions/bintools/jit_call_x64.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -389,7 +389,7 @@ inline jit_uint8_t Write_PushPOD(JitWriter *jit, const SourceHook::PassInfo *inf
if (g_StackUsage + 8 < SCHAR_MAX)
X64_Mov_RmRSP_Disp8_Reg(jit, reg2, (jit_int8_t)g_StackUsage + 8);
else
X64_Mov_RmRSP_Disp32_Reg(jit, reg2, (jit_int8_t)g_StackUsage + 8);
X64_Mov_RmRSP_Disp32_Reg(jit, reg2, g_StackUsage + 8);
g_StackUsage += 16;
} else {
g_StackUsage += 8;
Expand Down Expand Up @@ -470,25 +470,28 @@ inline void Write_PushFloat(JitWriter *jit, const SourceHook::PassInfo *info, un

if (!offset) {
X64_Movaps_Rm(jit, floatReg, kREG_EBX);
X64_Movups_Rm_Disp8(jit, floatReg2, kREG_EBX, offset+8);
} else if (offset < SCHAR_MAX) {
if (offset % 16 == 0)
X64_Movaps_Rm_Disp8(jit, floatReg, kREG_EBX, (jit_int8_t)offset);
else
X64_Movups_Rm_Disp8(jit, floatReg, kREG_EBX, (jit_int8_t)offset);
if ((offset + 8) % 16 == 0)
X64_Movaps_Rm_Disp8(jit, floatReg2, kREG_EBX, (jit_int8_t)offset+8);
else
X64_Movups_Rm_Disp8(jit, floatReg2, kREG_EBX, (jit_int8_t)offset+8);
} else {
if (offset % 16 == 0)
X64_Movaps_Rm_Disp32(jit, floatReg, kREG_EBX, offset);
else
X64_Movups_Rm_Disp32(jit, floatReg, kREG_EBX, offset);
}

if (offset + 8 < SCHAR_MAX) {
if ((offset + 8) % 16 == 0)
X64_Movaps_Rm_Disp8(jit, floatReg2, kREG_EBX, (jit_int8_t)(offset + 8));
else
X64_Movups_Rm_Disp8(jit, floatReg2, kREG_EBX, (jit_int8_t)(offset + 8));
} else {
if ((offset + 8) % 16 == 0)
X64_Movaps_Rm_Disp32(jit, floatReg2, kREG_EBX, offset+8);
X64_Movaps_Rm_Disp32(jit, floatReg2, kREG_EBX, offset + 8);
else
X64_Movups_Rm_Disp32(jit, floatReg2, kREG_EBX, offset+8);
X64_Movups_Rm_Disp32(jit, floatReg2, kREG_EBX, offset + 8);
}
}
} else if (info->flags & PASSFLAG_BYREF) {
Expand Down Expand Up @@ -529,9 +532,9 @@ inline void Write_PushFloat(JitWriter *jit, const SourceHook::PassInfo *info, un
X64_Movups_Rm_Disp8_Reg(jit, kREG_RSP, floatReg2, (jit_int8_t)g_StackUsage+8);
} else {
if ((g_StackUsage + 8) % 16 == 0)
X64_Movaps_Rm_Disp32_Reg(jit, kREG_RSP, floatReg, g_StackUsage+8);
X64_Movaps_Rm_Disp32_Reg(jit, kREG_RSP, floatReg2, g_StackUsage+8);
else
X64_Movups_Rm_Disp32_Reg(jit, kREG_RSP, floatReg, g_StackUsage+8);
X64_Movups_Rm_Disp32_Reg(jit, kREG_RSP, floatReg2, g_StackUsage+8);
}
g_StackUsage += 16;
} else {
Expand Down Expand Up @@ -566,9 +569,9 @@ inline uint8_t MapFloatToIntReg(uint8_t floatReg)

inline void Write_VarArgFloatCopy(JitWriter *jit, const SourceHook::PassInfo *info, uint8_t *floatRegs)
{
if (!floatRegs || floatRegs[0] == STACK_PARAM)
if (!floatRegs || MapFloatToIntReg(floatRegs[0]) == INVALID_REG)
return;

uint8_t intReg1 = MapFloatToIntReg(floatRegs[0]);

switch (info->size)
Expand All @@ -585,11 +588,11 @@ inline void Write_VarArgFloatCopy(JitWriter *jit, const SourceHook::PassInfo *in
//movq intReg1, floatReg[0]
//movq intReg2, floatReg[1]
X64_Movq_Reg_Xmm(jit, intReg1, floatRegs[0]);
if (floatRegs[1] != STACK_PARAM)
if (MapFloatToIntReg(floatRegs[1]) != INVALID_REG)
{
int intReg2 = MapFloatToIntReg(floatRegs[1]);
X64_Movq_Reg_Xmm(jit, intReg2, floatRegs[1]);
}
}
break;
}
}
Expand Down
38 changes: 23 additions & 15 deletions extensions/bintools/x64_macros.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ inline void X64_Movzx_Reg64_Rm8_Disp32(JitWriter *jit, jit_uint8_t dest, jit_uin

inline void X64_Mov_RmRSP_Reg(JitWriter *jit, jit_uint8_t src)
{
jit->write_ubyte(x64_rex(true, kREG_RSP, 0, src));
jit->write_ubyte(x64_rex(true, src, 0, kREG_RSP));
jit->write_ubyte(IA32_MOV_RM_REG);
jit->write_ubyte(ia32_modrm(MOD_MEM_REG, src & 7, kREG_RSP));
jit->write_ubyte(ia32_sib(NOSCALE, kREG_NOIDX, kREG_RSP));
Expand All @@ -185,13 +185,13 @@ inline void X64_Mov_RmRSP_Disp8_Reg(JitWriter *jit, jit_uint8_t src, jit_int8_t
jit->write_byte(disp);
}

inline void X64_Mov_RmRSP_Disp32_Reg(JitWriter *jit, jit_uint8_t src, jit_int8_t disp)
inline void X64_Mov_RmRSP_Disp32_Reg(JitWriter *jit, jit_uint8_t src, jit_int32_t disp)
{
jit->write_ubyte(x64_rex(true, src, 0, kREG_RSP));
jit->write_ubyte(IA32_MOV_RM_REG);
jit->write_ubyte(ia32_modrm(MOD_DISP32, src & 7, kREG_RSP));
jit->write_ubyte(ia32_sib(NOSCALE, kREG_NOIDX, kREG_RSP));
jit->write_byte(disp);
jit->write_int32(disp);
}

inline void X64_Movzx_Reg64_Rm16(JitWriter *jit, jit_uint8_t dest, jit_uint8_t src, jit_uint8_t mode)
Expand Down Expand Up @@ -312,6 +312,8 @@ inline void X64_Movaps_Rm_Disp8_Reg(JitWriter *jit, jit_uint8_t dest, jit_uint8_
jit->write_ubyte(0x0F);
jit->write_ubyte(0x29);
jit->write_ubyte(ia32_modrm(MOD_DISP8, src & 7, dest & 7));
if ((dest & 7) == kREG_RSP)
jit->write_ubyte(ia32_sib(NOSCALE, kREG_NOIDX, dest & 7));
jit->write_byte(disp);
}

Expand All @@ -322,6 +324,8 @@ inline void X64_Movups_Rm_Disp8_Reg(JitWriter *jit, jit_uint8_t dest, jit_uint8_
jit->write_ubyte(0x0F);
jit->write_ubyte(0x11);
jit->write_ubyte(ia32_modrm(MOD_DISP8, src & 7, dest & 7));
if ((dest & 7) == kREG_RSP)
jit->write_ubyte(ia32_sib(NOSCALE, kREG_NOIDX, dest & 7));
jit->write_byte(disp);
}

Expand All @@ -332,7 +336,9 @@ inline void X64_Movaps_Rm_Disp32_Reg(JitWriter *jit, jit_uint8_t dest, jit_uint8
jit->write_ubyte(0x0F);
jit->write_ubyte(0x29);
jit->write_ubyte(ia32_modrm(MOD_DISP32, src & 7, dest & 7));
jit->write_byte(disp);
if ((dest & 7) == kREG_RSP)
jit->write_ubyte(ia32_sib(NOSCALE, kREG_NOIDX, dest & 7));
jit->write_int32(disp);
}

inline void X64_Movups_Rm_Disp32_Reg(JitWriter *jit, jit_uint8_t dest, jit_uint8_t src, jit_int32_t disp)
Expand All @@ -342,7 +348,9 @@ inline void X64_Movups_Rm_Disp32_Reg(JitWriter *jit, jit_uint8_t dest, jit_uint8
jit->write_ubyte(0x0F);
jit->write_ubyte(0x11);
jit->write_ubyte(ia32_modrm(MOD_DISP32, src & 7, dest & 7));
jit->write_byte(disp);
if ((dest & 7) == kREG_RSP)
jit->write_ubyte(ia32_sib(NOSCALE, kREG_NOIDX, dest & 7));
jit->write_int32(disp);
}

inline void X64_Movlps_Rm(JitWriter *jit, jit_uint8_t dest, jit_uint8_t src)
Expand Down Expand Up @@ -410,7 +418,7 @@ inline void X64_Movaps_Rm_Disp32(JitWriter *jit, jit_uint8_t dest, jit_uint8_t s
jit->write_ubyte(0x0F);
jit->write_ubyte(0x28);
jit->write_ubyte(ia32_modrm(MOD_DISP32, dest & 7, src & 7));
jit->write_byte(disp);
jit->write_int32(disp);
}

inline void X64_Movups_Rm_Disp32(JitWriter *jit, jit_uint8_t dest, jit_uint8_t src, jit_int32_t disp)
Expand All @@ -420,34 +428,34 @@ inline void X64_Movups_Rm_Disp32(JitWriter *jit, jit_uint8_t dest, jit_uint8_t s
jit->write_ubyte(0x0F);
jit->write_ubyte(0x10);
jit->write_ubyte(ia32_modrm(MOD_DISP32, dest & 7, src & 7));
jit->write_byte(disp);
jit->write_int32(disp);
}

inline void X64_Movapd_Rm(JitWriter *jit, jit_uint8_t dest, jit_uint8_t src)
{
jit->write_ubyte(0x66);
if (dest >= kREG_XMM8 || src >= kREG_XMM8)
jit->write_ubyte(x64_rex(false, dest, 0, src));
jit->write_ubyte(0x66);
jit->write_ubyte(0x0F);
jit->write_ubyte(0x28);
jit->write_ubyte(ia32_modrm(MOD_MEM_REG, dest & 7, src & 7));
}

inline void X64_Movupd_Rm(JitWriter *jit, jit_uint8_t dest, jit_uint8_t src)
{
jit->write_ubyte(0x66);
if (dest >= kREG_XMM8 || src >= kREG_XMM8)
jit->write_ubyte(x64_rex(false, dest, 0, src));
jit->write_ubyte(0x66);
jit->write_ubyte(0x0F);
jit->write_ubyte(0x10);
jit->write_ubyte(ia32_modrm(MOD_MEM_REG, dest & 7, src & 7));
}

inline void X64_Movapd_Rm_Disp8(JitWriter *jit, jit_uint8_t dest, jit_uint8_t src, jit_int8_t disp)
{
jit->write_ubyte(0x66);
if (dest >= kREG_XMM8 || src >= kREG_XMM8)
jit->write_ubyte(x64_rex(false, dest, 0, src));
jit->write_ubyte(0x66);
jit->write_ubyte(0x0F);
jit->write_ubyte(0x28);
jit->write_ubyte(ia32_modrm(MOD_DISP8, dest & 7, src & 7));
Expand All @@ -456,9 +464,9 @@ inline void X64_Movapd_Rm_Disp8(JitWriter *jit, jit_uint8_t dest, jit_uint8_t sr

inline void X64_Movupd_Rm_Disp8(JitWriter *jit, jit_uint8_t dest, jit_uint8_t src, jit_int8_t disp)
{
jit->write_ubyte(0x66);
if (dest >= kREG_XMM8 || src >= kREG_XMM8)
jit->write_ubyte(x64_rex(false, dest, 0, src));
jit->write_ubyte(0x66);
jit->write_ubyte(0x0F);
jit->write_ubyte(0x10);
jit->write_ubyte(ia32_modrm(MOD_DISP8, dest & 7, src & 7));
Expand All @@ -467,24 +475,24 @@ inline void X64_Movupd_Rm_Disp8(JitWriter *jit, jit_uint8_t dest, jit_uint8_t sr

inline void X64_Movapd_Rm_Disp32(JitWriter *jit, jit_uint8_t dest, jit_uint8_t src, jit_int32_t disp)
{
jit->write_ubyte(0x66);
if (dest >= kREG_XMM8 || src >= kREG_XMM8)
jit->write_ubyte(x64_rex(false, dest, 0, src));
jit->write_ubyte(0x66);
jit->write_ubyte(0x0F);
jit->write_ubyte(0x28);
jit->write_ubyte(ia32_modrm(MOD_DISP32, dest & 7, src & 7));
jit->write_byte(disp);
jit->write_int32(disp);
}

inline void X64_Movupd_Rm_Disp32(JitWriter *jit, jit_uint8_t dest, jit_uint8_t src, jit_int32_t disp)
{
jit->write_ubyte(0x66);
if (dest >= kREG_XMM8 || src >= kREG_XMM8)
jit->write_ubyte(x64_rex(false, dest, 0, src));
jit->write_ubyte(0x66);
jit->write_ubyte(0x0F);
jit->write_ubyte(0x10);
jit->write_ubyte(ia32_modrm(MOD_DISP32, dest & 7, src & 7));
jit->write_byte(disp);
jit->write_int32(disp);
}

inline void X64_Cld(JitWriter *jit)
Expand Down