Using Manifests for build

.github/workflows/build.yml

@@ -35,7 +35,7 @@ jobs:
 
         sm_version:
           - "latest"
-          - "1.11"
+          - "1.12"
 
         include:
           - mm_version: "latest"
@@ -44,8 +44,8 @@ jobs:
           - sm_version: "latest"
             sm_branch: "master"
 
-          - sm_version: "1.11"
-            sm_branch: "1.11-dev"
+          - sm_version: "1.12"
+            sm_branch: "1.12-dev"
 
           - os: ubuntu-latest
             os_short: linux
@@ -131,6 +131,19 @@ jobs:
           ref: csgo
           path: hl2sdk-csgo
 
+      - name: Checking out hl2sdk-hl2dm
+        uses: actions/checkout@v4
+        with:
+          repository: alliedmodders/hl2sdk
+          ref: hl2dm
+          path: hl2sdk-hl2dm
+
+      - name: Checking out hl2sdk-manifests
+        uses: actions/checkout@v4
+        with:
+          repository: alliedmodders/hl2sdk-manifests
+          path: hl2sdk-manifests
+
       - name: Checking out own repository
         uses: actions/checkout@v4
         with:
@@ -142,7 +155,7 @@ jobs:
         run: |
           mkdir build
           cd build
-          python ../configure.py --enable-optimize --sm-path="${{ github.workspace }}/sourcemod-${{ matrix.sm_version }}" --mms-path="${{ github.workspace }}/metamod-${{ matrix.mm_version }}" --targets=x86,x86_64
+          python ../configure.py --enable-optimize --sm-path="${{ github.workspace }}/sourcemod-${{ matrix.sm_version }}" --hl2sdk-manifest-path="${{ github.workspace }}/hl2sdk-manifests" --mms-path="${{ github.workspace }}/metamod-${{ matrix.mm_version }}" --targets=x86,x86_64
           ambuild
 
       - name: Uploading package

AMBuildScript

@@ -1,69 +1,6 @@
 # vim: set sts=2 ts=8 sw=2 tw=99 et ft=python:
 import os, sys, shutil
 
-class SDK(object):
-  def __init__(self, sdk, ext, aDef, name, platform, dir):
-    self.folder = 'hl2sdk-' + dir
-    self.envvar = sdk
-    self.ext = ext
-    self.code = aDef
-    self.define = name
-    self.platform = platform
-    self.name = dir
-    self.path = None # Actual path
-    self.platformSpec = platform
-
-    # By default, nothing supports x64.
-    if type(platform) is list:
-      self.platformSpec = {p: ['x86'] for p in platform}
-    else:
-      self.platformSpec = platform
-
-  def shouldBuild(self, targets):
-    for cxx in targets:
-      if cxx.target.platform in self.platformSpec:
-        if cxx.target.arch in self.platformSpec[cxx.target.platform]:
-          return True
-    return False
-
-WinOnly = ['windows']
-WinLinux = ['windows', 'linux']
-WinLinuxMac = ['windows', 'linux', 'mac']
-CSGO = {
-  'windows': ['x86'],
-  'linux': ['x86', 'x86_64'],
-  'mac': ['x86_64']
-}
-TF2 = {
-  'windows': ['x86', 'x86_64'],
-  'linux': ['x86', 'x86_64'],
-  'mac': ['x86']
-}
-
-PossibleSDKs = {
-  # 'episode1':  SDK('HL2SDK', '2.ep1', '1', 'EPISODEONE', WinLinux, 'episode1'),
-  # 'ep2':  SDK('HL2SDKOB', '2.ep2', '3', 'ORANGEBOX', WinLinux, 'orangebox'),
-  'css':  SDK('HL2SDKCSS', '2.css', '6', 'CSS', WinLinuxMac, 'css'),
-  # 'hl2dm':  SDK('HL2SDKHL2DM', '2.hl2dm', '7', 'HL2DM', WinLinuxMac, 'hl2dm'),
-  # 'dods': SDK('HL2SDKDODS', '2.dods', '8', 'DODS', WinLinuxMac, 'dods'),
-  # 'sdk2013': SDK('HL2SDK2013', '2.sdk2013', '9', 'SDK2013', WinLinuxMac, 'sdk2013'),
-  'tf2':  SDK('HL2SDKTF2', '2.tf2', '11', 'TF2', TF2, 'tf2'),
-  'l4d':  SDK('HL2SDKL4D', '2.l4d', '12', 'LEFT4DEAD', WinLinuxMac, 'l4d'),
-  # 'nucleardawn': SDK('HL2SDKND', '2.nd', '13', 'NUCLEARDAWN', WinLinuxMac, 'nucleardawn'),
-  'l4d2': SDK('HL2SDKL4D2', '2.l4d2', '15', 'LEFT4DEAD2', WinLinuxMac, 'l4d2'),
-  # 'darkm':  SDK('HL2SDK-DARKM', '2.darkm', '2', 'DARKMESSIAH', WinOnly, 'darkm'),
-  # 'swarm':  SDK('HL2SDK-SWARM', '2.swarm', '16', 'ALIENSWARM', WinOnly, 'swarm'),
-  # 'bgt':  SDK('HL2SDK-BGT', '2.bgt', '4', 'BLOODYGOODTIME', WinOnly, 'bgt'),
-  # 'eye':  SDK('HL2SDK-EYE', '2.eye', '5', 'EYE', WinOnly, 'eye'),
-  'csgo': SDK('HL2SDKCSGO', '2.csgo', '21', 'CSGO', CSGO, 'csgo'),
-  # 'portal2':  SDK('HL2SDKPORTAL2', '2.portal2', '17', 'PORTAL2', [], 'portal2'),
-  # 'blade':  SDK('HL2SDKBLADE', '2.blade', '18', 'BLADE', WinOnly, 'blade'),
-  # 'insurgency':  SDK('HL2SDKINSURGENCY', '2.insurgency', '19', 'INSURGENCY', WinLinuxMac, 'insurgency'),
-  # 'contagion':  SDK('HL2SDKCONTAGION', '2.contagion', '14', 'CONTAGION', WinOnly, 'contagion'),
-  # 'bms':  SDK('HL2SDKBMS', '2.bms', '10', 'BMS', WinLinux, 'bms'),
-  # 'doi':  SDK('HL2SDKDOI', '2.doi', '20', 'DOI', WinLinuxMac, 'doi'),
-}
-
 def ResolveEnvPath(env, folder):
   if env in os.environ:
     path = os.environ[env]
@@ -84,7 +21,7 @@ def ResolveEnvPath(env, folder):
 
 def Normalize(path):
   return os.path.abspath(os.path.normpath(path))
-  
+
 def SetArchFlags(compiler):
   if compiler.behavior == 'gcc':
     if compiler.target.arch == 'x86_64':
@@ -93,10 +30,31 @@ def SetArchFlags(compiler):
     if compiler.target.arch == 'x86_64':
       compiler.defines += ['WIN64']
 
+hl2sdk_manifests_path = None
+
+# First we check if the manifest exists in the current path
+hl2sdk_manifests_path = os.path.join(builder.sourcePath, 'hl2sdk-manifests/SdkHelpers.ambuild')
+
+if not os.path.exists(hl2sdk_manifests_path):
+  # manifests does not exists in the project file, use the path from --hl2sdk-manifest-path
+  if not builder.options.hl2sdk_manifest:
+    raise Exception('HL2SDK Manifest root path not set! (--hl2sdk-manifest-path)')
+  else:
+    hl2sdk_manifests_path = os.path.join(builder.options.hl2sdk_manifest, 'SdkHelpers.ambuild')
+
+    if not os.path.exists(hl2sdk_manifests_path):
+      raise Exception('Could not find SdkHelpers.ambuild in the given HL2SDK Manifest path!')
+
+
+SdkHelpers = builder.Eval(hl2sdk_manifests_path, {
+    'Project': 'Cleaner'
+})
+
 class ExtensionConfig(object):
   def __init__(self):
+    self.sdk_manifests = []
     self.sdks = {}
-    self.binaries = []
+    self.sdk_targets = []
     self.extensions = []
     self.generated_headers = None
     self.mms_root = None
@@ -108,9 +66,7 @@ class ExtensionConfig(object):
     if builder.options.targets:
       target_archs = builder.options.targets.split(',')
     else:
-      target_archs = ['x86']
-      if builder.backend != 'amb2':
-        target_archs.append('x86_64')
+      target_archs = ['x86', 'x86_64']
 
     for arch in target_archs:
         try:
@@ -133,29 +89,26 @@ class ExtensionConfig(object):
       return 'Debug'
     return 'Release'
 
+  def findSdkPath(self, sdk_name):
+    dir_name = 'hl2sdk-{}'.format(sdk_name)
+    if builder.options.hl2sdk_root:
+      sdk_path = os.path.join(builder.options.hl2sdk_root, dir_name)
+      if os.path.exists(sdk_path):
+        return sdk_path
+    return ResolveEnvPath('HL2SDK{}'.format(sdk_name.upper()), dir_name)
+
+  def shouldIncludeSdk(self, sdk):
+    return not sdk.get('source2', False)
+
   def detectSDKs(self):
-    sdk_list = builder.options.sdks.split(',')
-    use_none = sdk_list[0] == 'none'
-    use_all = sdk_list[0] == 'all'
-    use_present = sdk_list[0] == 'present'
+    sdk_list = [s for s in builder.options.sdks.split(',') if s]
+    SdkHelpers.sdk_filter = self.shouldIncludeSdk
+    SdkHelpers.find_sdk_path = self.findSdkPath
+    SdkHelpers.findSdks(builder, self.all_targets, sdk_list)
 
-    for sdk_name in PossibleSDKs:
-      sdk = PossibleSDKs[sdk_name]
-      if sdk.shouldBuild(self.all_targets):
-        if builder.options.hl2sdk_root:
-          sdk_path = os.path.join(builder.options.hl2sdk_root, sdk.folder)
-        else:
-          sdk_path = ResolveEnvPath(sdk.envvar, sdk.folder)
-        if sdk_path is None or not os.path.isdir(sdk_path):
-          if use_all or sdk_name in sdk_list:
-            raise Exception('Could not find a valid path for {0}'.format(sdk.envvar))
-          continue
-        if use_all or use_present or sdk_name in sdk_list:
-          sdk.path = Normalize(sdk_path)
-          self.sdks[sdk_name] = sdk
-
-    if len(self.sdks) < 1 and len(sdk_list) and not use_none:
-      raise Exception('No applicable SDKs were found, nothing to do')
+    self.sdks = SdkHelpers.sdks
+    self.sdk_manifests = SdkHelpers.sdk_manifests
+    self.sdk_targets = SdkHelpers.sdk_targets
 
     if builder.options.sm_path:
       self.sm_root = builder.options.sm_path
@@ -184,7 +137,10 @@ class ExtensionConfig(object):
     self.mms_root = Normalize(self.mms_root)
 
   def configure(self):
-    if not set(self.target_archs).issubset(['x86', 'x86_64']):
+
+    allowed_archs = ['x86','x86_64']
+
+    if not set(self.target_archs).issubset(allowed_archs):
       raise Exception('Unknown target architecture: {0}'.format(self.target_archs))
 
     for cxx in self.all_targets:
@@ -206,7 +162,7 @@ class ExtensionConfig(object):
     elif cxx.family == 'msvc':
       self.configure_msvc(cxx)
 
-    # Optimizaiton
+    # Optimization
     if builder.options.opt == '1':
       cxx.defines += ['NDEBUG']
 
@@ -222,11 +178,6 @@ class ExtensionConfig(object):
     elif cxx.target.platform == 'windows':
       self.configure_windows(cxx)
 
-    # Finish up.
-    cxx.includes += [
-      os.path.join(self.sm_root, 'public'),
-    ]
-
   def configure_gcc(self, cxx):
     cxx.defines += [
       'stricmp=strcasecmp',
@@ -244,18 +195,21 @@ class ExtensionConfig(object):
       '-Wno-unused',
       '-Wno-switch',
       '-Wno-array-bounds',
+      '-Wno-unknown-pragmas',
+      '-Wno-dangling-else',
       '-fvisibility=hidden',
     ]
+
     if cxx.target.arch in ['x86', 'x86_64']:
       cxx.cflags += ['-msse']
 
     cxx.cxxflags += [
-      '-std=c++17',
       '-fno-threadsafe-statics',
       '-Wno-non-virtual-dtor',
       '-Wno-overloaded-virtual',
       '-Wno-register',
       '-fvisibility-inlines-hidden',
+      '-std=c++17',
     ]
 
     have_gcc = cxx.family == 'gcc'
@@ -349,12 +303,13 @@ class ExtensionConfig(object):
     cxx.cflags += ['/Oy-']
 
   def configure_linux(self, cxx):
-    cxx.defines += ['_LINUX', 'POSIX', '_FILE_OFFSET_BITS=64']
+    cxx.defines += ['LINUX', '_LINUX', 'POSIX', '_FILE_OFFSET_BITS=64']
     cxx.linkflags += ['-lm']
     if cxx.family == 'gcc':
       cxx.linkflags += ['-static-libgcc']
     elif cxx.family == 'clang':
       cxx.linkflags += ['-lgcc_eh']
+    cxx.linkflags += ['-static-libstdc++']
 
   def configure_mac(self, cxx):
     cxx.defines += ['OSX', '_OSX', 'POSIX', 'KE_ABSOLUTELY_NO_STL']
@@ -373,6 +328,7 @@ class ExtensionConfig(object):
     binary = compiler.Library(name)
     if binary.compiler.like('msvc'):
       binary.compiler.linkflags += ['/SUBSYSTEM:WINDOWS']
+    self.AddCxxCompat(binary)
     return binary
 
   def ProgramBuilder(self, compiler, name):
@@ -409,6 +365,7 @@ class ExtensionConfig(object):
     compiler.cxxincludes += [
       os.path.join(context.currentSourcePath),
       os.path.join(context.currentSourcePath, 'sdk'),
+      os.path.join(self.sm_root, 'public'),
       os.path.join(self.sm_root, 'public', 'extensions'),
       os.path.join(self.sm_root, 'sourcepawn', 'include'),
       os.path.join(self.sm_root, 'public', 'amtl', 'amtl'),
@@ -422,6 +379,12 @@ class ExtensionConfig(object):
     self.ConfigureForExtension(context, binary.compiler)
     return binary
 
+  def AddCxxCompat(self, binary):
+    if binary.compiler.target.platform == 'linux':
+      binary.sources += [
+        os.path.join(self.sm_root, 'public', 'amtl', 'compat', 'stdcxx.cpp'),
+      ]
+
   def ConfigureForHL2(self, context, binary, sdk):
     compiler = binary.compiler
     SetArchFlags(compiler)
@@ -431,133 +394,10 @@ class ExtensionConfig(object):
       os.path.join(self.mms_root, 'core', 'sourcehook'),
     ]
 
-    defines = ['SE_' + PossibleSDKs[i].define + '=' + PossibleSDKs[i].code for i in PossibleSDKs]
-    compiler.defines += defines
+    for other_sdk in self.sdk_manifests:
+      compiler.defines += ['SE_{}={}'.format(other_sdk['define'], other_sdk['code'])]
 
-    paths = [
-      ['public'],
-      ['public', 'engine'],
-      ['public', 'mathlib'],
-      ['public', 'vstdlib'],
-      ['public', 'tier0'],
-      ['public', 'tier1']
-    ]
-    if sdk.name == 'episode1' or sdk.name == 'darkm':
-      paths.append(['public', 'dlls'])
-      paths.append(['game_shared'])
-    else:
-      paths.append(['public', 'game', 'server'])
-      paths.append(['public', 'toolframework'])
-      paths.append(['game', 'shared'])
-      paths.append(['common'])
-
-    compiler.defines += ['SOURCE_ENGINE=' + sdk.code]
-
-    if sdk.name in ['sdk2013', 'bms'] and compiler.like('gcc'):
-      # The 2013 SDK already has these in public/tier0/basetypes.h
-      compiler.defines.remove('stricmp=strcasecmp')
-      compiler.defines.remove('_stricmp=strcasecmp')
-      compiler.defines.remove('_snprintf=snprintf')
-      compiler.defines.remove('_vsnprintf=vsnprintf')
-
-    if compiler.like('msvc'):
-      compiler.defines += ['COMPILER_MSVC']
-      if compiler.target.arch == 'x86':
-        compiler.defines += ['COMPILER_MSVC32']
-      elif compiler.target.arch == 'x86_64':
-        compiler.defines += ['COMPILER_MSVC64']
-      compiler.linkflags += ['legacy_stdio_definitions.lib']
-    else:
-      compiler.defines += ['COMPILER_GCC']
-
-    if compiler.target.arch == 'x86_64':
-      compiler.defines += ['X64BITS', 'PLATFORM_64BITS']
-
-    # For everything after Swarm, this needs to be defined for entity networking
-    # to work properly with sendprop value changes.
-    if sdk.name in ['blade', 'insurgency', 'doi', 'csgo']:
-      compiler.defines += ['NETWORK_VARS_ENABLED']
-
-    if sdk.name in ['css', 'hl2dm', 'dods', 'sdk2013', 'bms', 'tf2', 'l4d', 'nucleardawn', 'l4d2']:
-      if compiler.target.platform in ['linux', 'mac']:
-        compiler.defines += ['NO_HOOK_MALLOC', 'NO_MALLOC_OVERRIDE']
-
-    if compiler.target.platform == 'linux':
-      if sdk.name in ['csgo', 'blade']:
-        compiler.defines += ['_GLIBCXX_USE_CXX11_ABI=0']
-
-    for path in paths:
-      compiler.cxxincludes += [os.path.join(sdk.path, *path)]
-
-    if compiler.target.platform == 'linux':
-      if sdk.name == 'episode1':
-        lib_folder = os.path.join(sdk.path, 'linux_sdk')
-      elif sdk.name in ['sdk2013', 'bms']:
-        lib_folder = os.path.join(sdk.path, 'lib', 'public', 'linux32')
-      elif compiler.target.arch == 'x86_64':
-        lib_folder = os.path.join(sdk.path, 'lib', 'linux64')
-      else:
-        lib_folder = os.path.join(sdk.path, 'lib', 'linux')
-    elif compiler.target.platform == 'mac':
-      if sdk.name in ['sdk2013', 'bms']:
-        lib_folder = os.path.join(sdk.path, 'lib', 'public', 'osx32')
-      elif compiler.target.arch == 'x86_64':
-        lib_folder = os.path.join(sdk.path, 'lib', 'osx64')
-      else:
-        lib_folder = os.path.join(sdk.path, 'lib', 'mac')
-
-    if compiler.target.platform in ['linux', 'mac']:
-      if sdk.name in ['sdk2013', 'bms'] or compiler.target.arch == 'x86_64':
-        compiler.postlink += [
-          os.path.join(lib_folder, 'tier1.a'),
-          os.path.join(lib_folder, 'mathlib.a')
-        ]
-      else:
-        compiler.postlink += [
-          os.path.join(lib_folder, 'tier1_i486.a'),
-          os.path.join(lib_folder, 'mathlib_i486.a')
-        ]
-
-      if sdk.name in ['blade', 'insurgency', 'doi', 'csgo']:
-        if compiler.target.arch == 'x86_64':
-          compiler.postlink += [os.path.join(lib_folder, 'interfaces.a')]
-        else:
-          compiler.postlink += [os.path.join(lib_folder, 'interfaces_i486.a')]
-
-    dynamic_libs = []
-    if compiler.target.platform == 'linux':
-      if sdk.name in ['css', 'hl2dm', 'dods', 'tf2', 'sdk2013', 'bms', 'nucleardawn', 'l4d2', 'insurgency', 'doi']:
-        dynamic_libs = ['libtier0_srv.so', 'libvstdlib_srv.so']
-      elif compiler.target.arch == 'x86_64' and sdk.name in ['csgo', 'blade']:
-        dynamic_libs = ['libtier0_client.so', 'libvstdlib_client.so']
-      elif sdk.name in ['l4d', 'blade', 'insurgency', 'doi', 'csgo']:
-        dynamic_libs = ['libtier0.so', 'libvstdlib.so']
-      else:
-        dynamic_libs = ['tier0_i486.so', 'vstdlib_i486.so']
-    elif compiler.target.platform == 'mac':
-      compiler.linkflags.append('-liconv')
-      dynamic_libs = ['libtier0.dylib', 'libvstdlib.dylib']
-    elif compiler.target.platform == 'windows':
-      libs = ['tier0', 'tier1', 'vstdlib', 'mathlib']
-      if sdk.name in ['swarm', 'blade', 'insurgency', 'doi', 'csgo']:
-        libs.append('interfaces')
-      for lib in libs:
-        if compiler.target.arch == 'x86':
-          lib_path = os.path.join(sdk.path, 'lib', 'public', lib) + '.lib'
-        elif compiler.target.arch == 'x86_64':
-          lib_path = os.path.join(sdk.path, 'lib', 'public', 'win64', lib) + '.lib'
-        compiler.linkflags.append(lib_path)
-
-    for library in dynamic_libs:
-      source_path = os.path.join(lib_folder, library)
-      output_path = os.path.join(binary.localFolder, library)
-
-      # Ensure the output path exists.
-      context.AddFolder(binary.localFolder)
-      output = context.AddSymlink(source_path, output_path)
-
-      compiler.weaklinkdeps += [output]
-      compiler.linkflags[0:0] = [library]
+    SdkHelpers.configureCxx(context, binary, sdk)
 
     return binary
 
@@ -598,12 +438,14 @@ class ExtensionConfig(object):
 
   def HL2Config(self, project, context, compiler, name, sdk):
     binary = project.Configure(compiler, name,
-                               '{0} - {1} {2}'.format(self.tag, sdk.name, compiler.target.arch))
+                               '{0} - {1} {2}'.format(self.tag, sdk['name'], compiler.target.arch))
+    self.AddCxxCompat(binary)
     return self.ConfigureForHL2(context, binary, sdk)
 
   def HL2ExtConfig(self, project, context, compiler, name, sdk):
     binary = project.Configure(compiler, name,
-                               '{0} - {1} {2}'.format(self.tag, sdk.name, compiler.target.arch))
+                               '{0} - {1} {2}'.format(self.tag, sdk['name'], compiler.target.arch))
+    self.AddCxxCompat(binary)
     self.ConfigureForHL2(context, binary, sdk)
     self.ConfigureForExtension(context, binary.compiler)
     return binary

AMBuilder

@@ -9,12 +9,14 @@ project.sources += [
 
 for sdk_name in Extension.sdks:
   sdk = Extension.sdks[sdk_name]
-  
+  if sdk['name'] in ['mock']:
+    continue
+
   for cxx in builder.targets:
-    if not cxx.target.arch in sdk.platformSpec[cxx.target.platform]:
+    if not cxx.target.arch in sdk['platforms'][cxx.target.platform]:
       continue
     
-    binary = Extension.HL2ExtConfig(project, builder, cxx, 'cleaner.ext.' + sdk.ext, sdk)
+    binary = Extension.HL2ExtConfig(project, builder, cxx, 'cleaner.ext.' + sdk['extension'], sdk)
     Extension.AddCDetour(binary)
 
 Extension.extensions += builder.Add(project)

configure.py

@@ -21,6 +21,8 @@ if ambuild_version.startswith('2.1'):
 parser = run.BuildParser(sourcePath=sys.path[0], api='2.2')
 parser.options.add_argument('--hl2sdk-root', type=str, dest='hl2sdk_root', default=None,
 		                   help='Root search folder for HL2SDKs')
+parser.options.add_argument('--hl2sdk-manifest-path', type=str, dest='hl2sdk_manifest', default=None,
+                       help='Path to HL2SDK Manifests')
 parser.options.add_argument('--mms-path', type=str, dest='mms_path', default=None,
                        help='Path to Metamod:Source')
 parser.options.add_argument('--sm-path', type=str, dest='sm_path', default=None,