Pass around some more info so we can nicely return Vectors

This commit is contained in:
rtldg 2025-03-03 09:59:08 +00:00
parent daccb88340
commit a998e5e087
3 changed files with 22 additions and 20 deletions

View File

@ -74,10 +74,12 @@ typedef struct DataTypeSized_s {
type = DATA_TYPE_POINTER;
size = 0;
custom_register = None;
custom_register2 = None;
}
DataType_t type;
size_t size;
Register_t custom_register;
Register_t custom_register2; // Currently only used for Vector returns on Linux64
} DataTypeSized_t;

View File

@ -90,26 +90,16 @@ x86_64SystemVDefault::x86_64SystemVDefault(std::vector<DataTypeSized_t> &vecArgT
}
}
// TODO: Figure out if we need to do something different for Linux...
// Special return type
if (m_returnType.custom_register == None && m_returnType.type == DATA_TYPE_OBJECT &&
// If size unknown, or doesn't fit on 1, 2, 4 or 8 bytes
// special place must have been allocated for it
(m_returnType.size != 1
&& m_returnType.size != 2
&& m_returnType.size != 4
&& m_returnType.size != 8)) {
for (std::uint8_t i = 0; i < num_reg && m_returnType.custom_register == None; i++) {
if (!used_reg[i]) {
m_returnType.custom_register = params_reg[i];
used_reg[i] = true;
}
// Couldn't find a free register, this is a big problem
if (m_returnType.custom_register == None) {
puts("Missing free register for return pointer");
return;
}
}
// TODO: Object return-type register is RDI, even when thiscall.
// inRDI = out-object-address & outRAX = inRDI
// thiscall: inRDI = out-object-address & inRSI = this
//
// Currently not handling object return types other than Vector, which would fit into XMM0 & XMM1.
//
// It'd be great if we had PassInfo here...
if (m_returnType.custom_register != XMM0 && m_returnType.custom_register2 != XMM1 && m_returnType.type == DATA_TYPE_OBJECT) {
puts("Return type is an OBJECT but not a Vector. We don't support this right now.");
return;
}
for (auto& arg : m_vecArgTypes) {

View File

@ -255,6 +255,15 @@ ICallingConvention *ConstructCallingConvention(HookSetup *setup)
returnType.size = 0;
// TODO: Add support for a custom return register.
returnType.custom_register = None;
returnType.custom_register2 = None;
#if defined(DYNAMICHOOKS_x86_64) && defined(PLATFORM_LINUX)
if (setup->returnType == ReturnType_Vector) {
returnType.size = 16;
returnType.custom_register = XMM0;
returnType.custom_register2 = XMM1;
}
#endif
#ifdef DYNAMICHOOKS_x86_64
if (setup->callConv == CallConv_THISCALL) {
@ -387,6 +396,7 @@ ReturnAction_t HandleDetour(HookType_t hookType, CHook* pDetour)
if (pWrapper->callConv == CallConv_THISCALL && pWrapper->thisType != ThisPointer_Ignore)
{
// The this pointer is implicitly always the first argument.
// TODO: Linux64 with an object return value can mean `this` is the second argument, but not relevant for now.
void *thisPtr = pDetour->GetArgument<void *>(0);
cell_t thisAddr = GetThisPtr(thisPtr, pWrapper->thisType);
pCallback->PushCell(thisAddr);