chore: remove vkd3d shader compilation

* It does not produce usable output for DirectX unfortunately

src/ObjCompiling/Shader/D3DShaderCompiler.cpp

@@ -11,136 +11,15 @@
 #include <utility>
 
 #ifdef _WIN32
-#include <Windows.h>
+#include <d3dcompiler.h>
-#else
-#include <dlfcn.h>
 #endif
 
 namespace
 {
-    // The types based on vkd3d which is licensed under the
+#ifdef _WIN32
-    // GNU Lesser General Public License as published by the Free Software Foundation
-    // All types are prefixed with OAT_ to not cause conflicts as the original definition may be included
-    // depending on the platform.
-    // Types were duplicated to be available on any platform and to not require the corresponding library
-    // to be installed
-
-    typedef int OAT_HRESULT;
-#define OAT_SUCCEEDED(hr) ((OAT_HRESULT)(hr) >= 0)
-#define OAT_FAILED(hr) ((OAT_HRESULT)(hr) < 0)
-
-    typedef unsigned int OAT_ULONG;
-    typedef unsigned long OAT_ULONG_PTR;
-    typedef OAT_ULONG_PTR OAT_SIZE_T;
-
-#define OAT_HRESULT_TYPEDEF(x) ((OAT_HRESULT)x)
-
-#define OAT_S_OK OAT_HRESULT_TYPEDEF(0)
-#define OAT_S_FALSE OAT_HRESULT_TYPEDEF(1)
-
-#define OAT_E_FAIL OAT_HRESULT_TYPEDEF(0x80004005)
-
-#define OAT_D3DCOMPILE_DEBUG 0x00000001
-#define OAT_D3DCOMPILE_SKIP_VALIDATION 0x00000002
-#define OAT_D3DCOMPILE_SKIP_OPTIMIZATION 0x00000004
-#define OAT_D3DCOMPILE_PACK_MATRIX_ROW_MAJOR 0x00000008
-#define OAT_D3DCOMPILE_PACK_MATRIX_COLUMN_MAJOR 0x00000010
-#define OAT_D3DCOMPILE_PARTIAL_PRECISION 0x00000020
-#define OAT_D3DCOMPILE_FORCE_VS_SOFTWARE_NO_OPT 0x00000040
-#define OAT_D3DCOMPILE_FORCE_PS_SOFTWARE_NO_OPT 0x00000080
-#define OAT_D3DCOMPILE_NO_PRESHADER 0x00000100
-#define OAT_D3DCOMPILE_AVOID_FLOW_CONTROL 0x00000200
-#define OAT_D3DCOMPILE_PREFER_FLOW_CONTROL 0x00000400
-#define OAT_D3DCOMPILE_ENABLE_STRICTNESS 0x00000800
-#define OAT_D3DCOMPILE_ENABLE_BACKWARDS_COMPATIBILITY 0x00001000
-#define OAT_D3DCOMPILE_IEEE_STRICTNESS 0x00002000
-#define OAT_D3DCOMPILE_OPTIMIZATION_LEVEL0 0x00004000
-#define OAT_D3DCOMPILE_OPTIMIZATION_LEVEL1 0x00000000
-#define OAT_D3DCOMPILE_OPTIMIZATION_LEVEL2 0x0000c000
-#define OAT_D3DCOMPILE_OPTIMIZATION_LEVEL3 0x00008000
-#define OAT_D3DCOMPILE_RESERVED16 0x00010000
-#define OAT_D3DCOMPILE_RESERVED17 0x00020000
-#define OAT_D3DCOMPILE_WARNINGS_ARE_ERRORS 0x00040000
-#define OAT_D3DCOMPILE_RESOURCES_MAY_ALIAS 0x00080000
-#define OAT_D3DCOMPILE_ENABLE_UNBOUNDED_DESCRIPTOR_TABLES 0x00100000
-#define OAT_D3DCOMPILE_ALL_RESOURCES_BOUND 0x00200000
-#define OAT_D3DCOMPILE_DEBUG_NAME_FOR_SOURCE 0x00400000
-#define OAT_D3DCOMPILE_DEBUG_NAME_FOR_BINARY 0x00800000
-
-#ifndef _WIN32
-#ifdef __x86_64__
-#define __stdcall __attribute__((ms_abi))
-#else
-#if (__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ >= 2)) || defined(__APPLE__)
-#define __stdcall __attribute__((__stdcall__)) __attribute__((__force_align_arg_pointer__))
-#else
-#define __stdcall __attribute__((__stdcall__))
-#endif
-#endif
-#endif
-#define STDMETHODCALLTYPE __stdcall
-
-    enum OAT_D3D_INCLUDE_TYPE : uint32_t
-    {
-        OAT_D3D_INCLUDE_LOCAL = 0,
-        OAT_D3D_INCLUDE_SYSTEM = 1
-    };
-
-    struct OAT_ID3DInclude
-    {
-        // clang-format off
-        virtual OAT_HRESULT STDMETHODCALLTYPE Open(OAT_D3D_INCLUDE_TYPE includeType, const char* fileName, const void* parentData, const void** data, unsigned int* size) = 0;
-        virtual OAT_HRESULT STDMETHODCALLTYPE Close(const void* data) = 0;
-        // clang-format on
-    };
-
-    struct OAT_GUID
-    {
-        unsigned int Data1;
-        unsigned short Data2;
-        unsigned short Data3;
-        unsigned char Data4[8];
-    };
-
-    typedef OAT_GUID OAT_IID;
-
-    struct OAT_IUnknown
-    {
-        virtual OAT_HRESULT STDMETHODCALLTYPE QueryInterface(const OAT_IID& riid, void** object) = 0;
-
-        virtual OAT_ULONG STDMETHODCALLTYPE AddRef() = 0;
-
-        virtual OAT_ULONG STDMETHODCALLTYPE Release() = 0;
-    };
-
-    struct OAT_ID3DBlob : OAT_IUnknown
-    {
-        virtual void* STDMETHODCALLTYPE GetBufferPointer() = 0;
-
-        virtual OAT_SIZE_T STDMETHODCALLTYPE GetBufferSize() = 0;
-    };
-
-    struct OAT_D3D_SHADER_MACRO
-    {
-        const char* Name;
-        const char* Definition;
-    };
-
-    typedef OAT_HRESULT(STDMETHODCALLTYPE* D3DCompile_t)(const void* data,
-                                                         OAT_SIZE_T dataSize,
-                                                         const char* filename,
-                                                         const OAT_D3D_SHADER_MACRO* defines,
-                                                         OAT_ID3DInclude* include,
-                                                         const char* entrypoint,
-                                                         const char* target,
-                                                         unsigned int flags,
-                                                         unsigned int effectFlags,
-                                                         OAT_ID3DBlob** shader,
-                                                         OAT_ID3DBlob** errorMessages);
-
     constexpr size_t MAX_SHADER_SIZE = 0x1900000u;
 
-    class ShaderIncludeHandler : public OAT_ID3DInclude
+    class ShaderIncludeHandler : public ID3DInclude
     {
     public:
         explicit ShaderIncludeHandler(ISearchPath& searchPath)
@@ -148,18 +27,17 @@ namespace
         {
         }
 
-        OAT_HRESULT
+        HRESULT __stdcall Open(D3D_INCLUDE_TYPE includeType, const char* fileName, const void* parentData, const void** data, unsigned int* size) override
-        STDMETHODCALLTYPE Open(OAT_D3D_INCLUDE_TYPE includeType, const char* fileName, const void* parentData, const void** data, unsigned int* size) override
         {
             const auto fullFileName = shader::GetSourceFileNameForShaderAssetName(fileName);
             auto file = m_search_path.Open(fullFileName);
             if (!file.IsOpen() || file.m_length <= 0)
-                return OAT_E_FAIL;
+                return E_FAIL;
 
             if (std::cmp_greater(file.m_length, MAX_SHADER_SIZE))
             {
                 con::error("Invalid shader source \"{}\": File too big: {}\n", fileName, file.m_length);
-                return OAT_E_FAIL;
+                return E_FAIL;
             }
 
             const auto shaderSize = static_cast<size_t>(file.m_length);
@@ -172,21 +50,21 @@ namespace
 
             m_file_buffers_in_use.push_back(std::move(shaderData));
 
-            return OAT_S_OK;
+            return S_OK;
         }
 
-        OAT_HRESULT STDMETHODCALLTYPE Close(const void* data) override
+        HRESULT STDMETHODCALLTYPE Close(const void* data) override
         {
             for (auto i = m_file_buffers_in_use.begin(); i != m_file_buffers_in_use.end(); ++i)
             {
                 if (i->get() == data)
                 {
                     m_file_buffers_in_use.erase(i);
-                    return OAT_S_OK;
+                    return S_OK;
                 }
             }
 
-            return OAT_E_FAIL;
+            return E_FAIL;
         }
 
     private:
@@ -196,28 +74,21 @@ namespace
 
     bool initialized = false;
     bool compilationAvailable = false;
-    D3DCompile_t d3dCompile = nullptr;
+    pD3DCompile d3dCompile = nullptr;
 
     void PrintInitializationFailedMessage()
     {
-#ifdef _WIN32
         con::warn("Could not initialize shader compilation. Make sure DirectX is installed on your machine if you want to make use of it.");
-#else
-#ifdef ARCH_x86
-#define REQUIRED_VKD3D "lib32-vkd3d"
-#else
-#define REQUIRED_VKD3D "vkd3d"
-#endif
-        con::warn("Could not initialize shader compilation. Make sure " REQUIRED_VKD3D " is installed if you want to make use of it.");
-#endif
     }
 
+#endif
+
     void InitializeShaderCompilation()
     {
+#ifdef _WIN32
         initialized = true;
 
-#ifdef _WIN32
+        const auto d3dCompiler = LoadLibraryA("d3dcompiler_47.dll");
-        const auto d3dCompiler = LoadLibraryA("D3DCOMPILER_47.dll");
         if (!d3dCompiler)
         {
             PrintInitializationFailedMessage();
@@ -231,24 +102,7 @@ namespace
             return;
         }
 
-        d3dCompile = reinterpret_cast<D3DCompile_t>(d3dCompileAddress);
+        d3dCompile = reinterpret_cast<pD3DCompile>(d3dCompileAddress);
-        compilationAvailable = true;
-#else
-        const auto libvkd3dUtils = dlopen("libvkd3d-utils.so", RTLD_NOW);
-        if (!libvkd3dUtils)
-        {
-            PrintInitializationFailedMessage();
-            return;
-        }
-
-        const auto d3dCompileAddress = dlsym(libvkd3dUtils, "D3DCompile");
-        if (!d3dCompileAddress)
-        {
-            PrintInitializationFailedMessage();
-            return;
-        }
-
-        d3dCompile = reinterpret_cast<D3DCompile_t>(d3dCompileAddress);
         compilationAvailable = true;
 #endif
     }
@@ -258,10 +112,14 @@ namespace shader
 {
     bool ShaderCompilationAvailable()
     {
+#ifdef _WIN32
         if (!initialized)
             InitializeShaderCompilation();
 
         return compilationAvailable;
+#else
+        return false;
+#endif
     }
 
     result::Expected<std::optional<CompiledShader>, std::string> CompileShader(const std::string& shaderFile,
@@ -271,6 +129,7 @@ namespace shader
                                                                                ISearchPath& searchPath,
                                                                                MemoryManager& memory)
     {
+#ifdef _WIN32
         if (!initialized)
             InitializeShaderCompilation();
         if (!compilationAvailable)
@@ -289,16 +148,16 @@ namespace shader
         file.m_stream->read(shaderData.get(), static_cast<std::streamsize>(shaderSize));
         file.m_stream.reset();
 
-        unsigned shaderFlags = OAT_D3DCOMPILE_OPTIMIZATION_LEVEL1;
+        unsigned shaderFlags = D3DCOMPILE_OPTIMIZATION_LEVEL1;
         if (debug)
-            shaderFlags |= OAT_D3DCOMPILE_DEBUG;
+            shaderFlags |= D3DCOMPILE_DEBUG;
 
         ShaderIncludeHandler shaderIncluder(searchPath);
 
-        OAT_ID3DBlob* shaderBlob = nullptr;
+        ID3DBlob* shaderBlob = nullptr;
-        OAT_ID3DBlob* errorBlob = nullptr;
+        ID3DBlob* errorBlob = nullptr;
         const auto errorCode = d3dCompile(shaderData.get(),
-                                          static_cast<OAT_SIZE_T>(shaderSize),
+                                          static_cast<SIZE_T>(shaderSize),
                                           fileName.c_str(),
                                           nullptr,
                                           &shaderIncluder,
@@ -309,7 +168,7 @@ namespace shader
                                           &shaderBlob,
                                           &errorBlob);
 
-        if (OAT_FAILED(errorCode))
+        if (FAILED(errorCode))
         {
             std::string errorMessage;
 
@@ -339,5 +198,8 @@ namespace shader
             .m_shader_bin = assetShaderBuffer,
             .m_shader_size = shaderBlobSize,
         });
+#else
+        return result::Unexpected<std::string>("Shader compilation unavailable");
+#endif
     }
 } // namespace shader