Pass around some more info so we can nicely return Vectors

This commit is contained in:
rtldg 2025-03-03 09:59:08 +00:00
parent daccb88340
commit a998e5e087
3 changed files with 22 additions and 20 deletions

View File

@ -74,10 +74,12 @@ typedef struct DataTypeSized_s {
type = DATA_TYPE_POINTER; type = DATA_TYPE_POINTER;
size = 0; size = 0;
custom_register = None; custom_register = None;
custom_register2 = None;
} }
DataType_t type; DataType_t type;
size_t size; size_t size;
Register_t custom_register; Register_t custom_register;
Register_t custom_register2; // Currently only used for Vector returns on Linux64
} DataTypeSized_t; } DataTypeSized_t;

View File

@ -90,26 +90,16 @@ x86_64SystemVDefault::x86_64SystemVDefault(std::vector<DataTypeSized_t> &vecArgT
} }
} }
// TODO: Figure out if we need to do something different for Linux... // TODO: Object return-type register is RDI, even when thiscall.
// Special return type // inRDI = out-object-address & outRAX = inRDI
if (m_returnType.custom_register == None && m_returnType.type == DATA_TYPE_OBJECT && // thiscall: inRDI = out-object-address & inRSI = this
// If size unknown, or doesn't fit on 1, 2, 4 or 8 bytes //
// special place must have been allocated for it // Currently not handling object return types other than Vector, which would fit into XMM0 & XMM1.
(m_returnType.size != 1 //
&& m_returnType.size != 2 // It'd be great if we had PassInfo here...
&& m_returnType.size != 4 if (m_returnType.custom_register != XMM0 && m_returnType.custom_register2 != XMM1 && m_returnType.type == DATA_TYPE_OBJECT) {
&& m_returnType.size != 8)) { puts("Return type is an OBJECT but not a Vector. We don't support this right now.");
for (std::uint8_t i = 0; i < num_reg && m_returnType.custom_register == None; i++) { return;
if (!used_reg[i]) {
m_returnType.custom_register = params_reg[i];
used_reg[i] = true;
}
// Couldn't find a free register, this is a big problem
if (m_returnType.custom_register == None) {
puts("Missing free register for return pointer");
return;
}
}
} }
for (auto& arg : m_vecArgTypes) { for (auto& arg : m_vecArgTypes) {

View File

@ -255,6 +255,15 @@ ICallingConvention *ConstructCallingConvention(HookSetup *setup)
returnType.size = 0; returnType.size = 0;
// TODO: Add support for a custom return register. // TODO: Add support for a custom return register.
returnType.custom_register = None; returnType.custom_register = None;
returnType.custom_register2 = None;
#if defined(DYNAMICHOOKS_x86_64) && defined(PLATFORM_LINUX)
if (setup->returnType == ReturnType_Vector) {
returnType.size = 16;
returnType.custom_register = XMM0;
returnType.custom_register2 = XMM1;
}
#endif
#ifdef DYNAMICHOOKS_x86_64 #ifdef DYNAMICHOOKS_x86_64
if (setup->callConv == CallConv_THISCALL) { if (setup->callConv == CallConv_THISCALL) {
@ -387,6 +396,7 @@ ReturnAction_t HandleDetour(HookType_t hookType, CHook* pDetour)
if (pWrapper->callConv == CallConv_THISCALL && pWrapper->thisType != ThisPointer_Ignore) if (pWrapper->callConv == CallConv_THISCALL && pWrapper->thisType != ThisPointer_Ignore)
{ {
// The this pointer is implicitly always the first argument. // The this pointer is implicitly always the first argument.
// TODO: Linux64 with an object return value can mean `this` is the second argument, but not relevant for now.
void *thisPtr = pDetour->GetArgument<void *>(0); void *thisPtr = pDetour->GetArgument<void *>(0);
cell_t thisAddr = GetThisPtr(thisPtr, pWrapper->thisType); cell_t thisAddr = GetThisPtr(thisPtr, pWrapper->thisType);
pCallback->PushCell(thisAddr); pCallback->PushCell(thisAddr);