From 9121472061d7a8f62f1ba48bf1950097218d8043 Mon Sep 17 00:00:00 2001
From: peace-maker <peace-maker@wcfan.de>
Date: Tue, 31 May 2022 14:58:36 +0200
Subject: [PATCH] DHooks: Error on argument passflags for detours (#1773)

The passflags are only supported by SourceHook for virtual hooks and are ignored for detours with DynamicDetours. This caused confusion, so throw an error when trying to set e.g. the DHookPass_ByRef flag on detour arguments.
---
 extensions/dhooks/natives.cpp    | 16 ++++++++++++++++
 extensions/dhooks/signatures.cpp | 14 ++++++++++++++
 extensions/dhooks/vhook.h        |  8 ++++++++
 3 files changed, 38 insertions(+)

diff --git a/extensions/dhooks/natives.cpp b/extensions/dhooks/natives.cpp
index e1ace0b40..8757e8319 100644
--- a/extensions/dhooks/natives.cpp
+++ b/extensions/dhooks/natives.cpp
@@ -256,6 +256,15 @@ cell_t Native_SetFromConf(IPluginContext *pContext, const cell_t *params)
 	setup->funcAddr = addr;
 	setup->offset = offset;
 
+	if (addr == nullptr)
+	{
+		setup->hookMethod = Virtual;
+	}
+	else
+	{
+		setup->hookMethod = Detour;
+	}
+
 	return 1;
 }
 
@@ -282,6 +291,13 @@ cell_t Native_AddParam(IPluginContext *pContext, const cell_t *params)
 		info.flags = PASSFLAG_BYVAL;
 	}
 
+	// DynamicDetours doesn't expose the passflags concept like SourceHook.
+	// See if we're trying to set some invalid flags on detour arguments.
+	if(setup->hookMethod == Detour && (info.flags & ~PASSFLAG_BYVAL) > 0)
+	{
+		return pContext->ThrowNativeError("Pass flags are only supported for virtual hooks.");
+	}
+
 	if (params[0] >= 5)
 	{
 		PluginRegister custom_register = (PluginRegister)params[5];
diff --git a/extensions/dhooks/signatures.cpp b/extensions/dhooks/signatures.cpp
index e5ddd8b9e..45b673cc3 100644
--- a/extensions/dhooks/signatures.cpp
+++ b/extensions/dhooks/signatures.cpp
@@ -376,6 +376,20 @@ SMCResult SignatureGameConfig::ReadSMC_LeavingSection(const SMCStates *states)
 			return SMCResult_HaltFail;
 		}
 
+		if (!g_CurrentSignature->offset.length())
+		{
+			// DynamicDetours doesn't expose the passflags concept like SourceHook.
+			// See if we're trying to set some invalid flags on detour arguments.
+			for (auto &arg : g_CurrentSignature->args)
+			{
+				if ((arg.info.flags & ~PASSFLAG_BYVAL) > 0)
+				{
+					smutils->LogError(myself, "Function \"%s\" uses unsupported pass flags in argument \"%s\". Flags are only supported for virtual hooks: line: %i col: %i", g_CurrentFunctionName.c_str(), arg.name.c_str(), states->line, states->col);
+					return SMCResult_HaltFail;
+				}
+			}
+		}
+
 		// Save this function signature in our cache.
 		signatures_.insert(g_CurrentFunctionName.c_str(), g_CurrentSignature);
 		g_CurrentFunctionName = "";
diff --git a/extensions/dhooks/vhook.h b/extensions/dhooks/vhook.h
index 3340049c4..6146d1ad6 100644
--- a/extensions/dhooks/vhook.h
+++ b/extensions/dhooks/vhook.h
@@ -225,6 +225,11 @@ public:
 	DHooksInfo *dg;
 };
 
+enum HookMethod {
+	Virtual,
+	Detour
+};
+
 class HookSetup
 {
 public:
@@ -238,6 +243,7 @@ public:
 		this->offset = offset;
 		this->funcAddr = nullptr;
 		this->callback = callback;
+		this->hookMethod = Virtual;
 	};
 	HookSetup(ReturnType returnType, unsigned int returnFlag, CallingConvention callConv, ThisPointerType thisType, void *funcAddr)
 	{
@@ -249,6 +255,7 @@ public:
 		this->offset = -1;
 		this->funcAddr = funcAddr;
 		this->callback = nullptr;
+		this->hookMethod = Detour;
 	};
 	~HookSetup(){};
 
@@ -266,6 +273,7 @@ public:
 	int offset;
 	void *funcAddr;
 	IPluginFunction *callback;
+	HookMethod hookMethod;
 };
 
 class DHooksManager