Add support for building engine target as shared library on all platforms

Source/Engine/Main/Windows/main.cpp

@@ -25,8 +25,6 @@ extern "C" {
 __declspec(dllexport) int32 AmdPowerXpressRequestHighPerformance = 1;
 }
 
-extern LONG CALLBACK SehExceptionHandler(EXCEPTION_POINTERS* ep);
-
 #if FLAX_TESTS
 int main(int argc, char* argv[])
 #else
@@ -54,7 +52,7 @@ int APIENTRY wWinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPTSTR lpCmd
     {
         return Engine::Main(lpCmdLine);
     }
-    __except (SehExceptionHandler(GetExceptionInformation()))
+    __except (Platform::SehExceptionHandler(GetExceptionInformation()))
     {
         return -1;
     }

Source/Engine/Platform/Win32/Win32Thread.cpp

@@ -88,10 +88,6 @@ bool Win32Thread::Start(uint32 stackSize)
     return false;
 }
 
-#if PLATFORM_WINDOWS
-extern LONG CALLBACK SehExceptionHandler(EXCEPTION_POINTERS* ep);
-#endif
-
 unsigned long Win32Thread::ThreadProc(void* pThis)
 {
     auto thread = (Win32Thread*)pThis;
@@ -103,7 +99,7 @@ unsigned long Win32Thread::ThreadProc(void* pThis)
         return static_cast<unsigned long>(exitCode);
     }
 #if PLATFORM_WINDOWS
-    __except (SehExceptionHandler(GetExceptionInformation()))
+    __except (Platform::SehExceptionHandler(GetExceptionInformation()))
     {
         return -1;
     }

Source/Engine/Platform/Windows/WindowsPlatform.cpp

@@ -273,7 +273,7 @@ LRESULT CALLBACK WndProc(HWND hwnd, UINT msg, WPARAM wParam, LPARAM lParam)
     return DefWindowProc(hwnd, msg, wParam, lParam);
 }
 
-LONG CALLBACK SehExceptionHandler(EXCEPTION_POINTERS* ep)
+long __stdcall WindowsPlatform::SehExceptionHandler(EXCEPTION_POINTERS* ep)
 {
     if (ep->ExceptionRecord->ExceptionCode == CLR_EXCEPTION)
     {

Source/Engine/Platform/Windows/WindowsPlatform.h

@@ -23,6 +23,9 @@ public:
     /// </summary>
     static void* Instance;
 
+    // Native exceptions handling function.
+    static long __stdcall SehExceptionHandler(struct _EXCEPTION_POINTERS* ep);
+
 public:
 
     /// <summary>

Source/Engine/Scripting/Runtime/DotNet.cpp

@@ -281,6 +281,10 @@ bool MCore::LoadEngine()
         flaxLibraryPath = ::String(StringUtils::GetDirectoryName(Platform::GetExecutableFilePath())) / StringUtils::GetFileName(flaxLibraryPath);
     }
 #endif
+    if (!FileSystem::FileExists(flaxLibraryPath))
+    {
+        LOG(Error, "Flax Engine native library file is missing ({0})", flaxLibraryPath);
+    }
     RegisterNativeLibrary("FlaxEngine", flaxLibraryPath.Get());
 
     MRootDomain = New<MDomain>("Root");

Source/Engine/Scripting/Scripting.Build.cs

@@ -43,7 +43,7 @@ public class Scripting : EngineModule
                 if (options.Target is EngineTarget engineTarget && engineTarget.UseSeparateMainExecutable(options))
                 {
                     // Build target doesn't support linking again main executable (eg. Linux) thus additional shared library is used for the engine (eg. libFlaxEditor.so)
-                    var fileName = options.Platform.GetLinkOutputFileName(engineTarget.OutputName, LinkerOutput.SharedLibrary);
+                    var fileName = options.Platform.GetLinkOutputFileName(EngineTarget.LibraryName, LinkerOutput.SharedLibrary);
                     options.CompileEnv.PreprocessorDefinitions.Add("MCORE_MAIN_MODULE_NAME=" + fileName);
                 }
             }

Source/Tools/Flax.Build/Build/EngineTarget.cs

@@ -16,6 +16,11 @@ namespace Flax.Build
     {
         private static Version _engineVersion;
 
+        /// <summary>
+        /// Name of the native engine library.
+        /// </summary>
+        public const string LibraryName = "FlaxEngine";
+
         /// <summary>
         /// Gets the engine project.
         /// </summary>
@@ -50,6 +55,11 @@ namespace Flax.Build
                 defines.Add(string.Format("FLAX_{0}_{1}_OR_NEWER", engineVersion.Major, minor));
         }
 
+        /// <summary>
+        /// True if target is built as monolithic executable with Main module inside, otherwise built as shared library with separate executable made of Main module only.
+        /// </summary>
+        public bool IsMonolithicExecutable = true;
+
         /// <inheritdoc />
         public override void Init()
         {
@@ -65,14 +75,23 @@ namespace Flax.Build
         /// <inheritdoc />
         public override string GetOutputFilePath(BuildOptions options, TargetOutputType? outputType)
         {
+            var useSeparateMainExe = UseSeparateMainExecutable(options);
+
             // If building engine executable for platform doesn't support referencing it when linking game shared libraries
-            if (outputType == null && UseSeparateMainExecutable(options))
+            if (outputType == null && useSeparateMainExe)
             {
                 // Build into shared library
                 outputType = TargetOutputType.Library;
             }
 
-            return base.GetOutputFilePath(options, outputType);
+            // Override output name to shared library name when building library for the separate main executable
+            var outputName = OutputName;
+            if (useSeparateMainExe && (outputType ?? OutputType) == TargetOutputType.Library)
+                OutputName = LibraryName;
+
+            var result = base.GetOutputFilePath(options, outputType);
+            OutputName = outputName;
+            return result;
         }
 
         /// <inheritdoc />
@@ -123,9 +142,13 @@ namespace Flax.Build
         /// <summary>
         /// Returns true if this build target should use separate (aka main-only) executable file and separate runtime (in shared library). Used on platforms that don't support linking again executable file but only shared library (see HasExecutableFileReferenceSupport).
         /// </summary>
-        public bool UseSeparateMainExecutable(BuildOptions buildOptions)
+        public virtual bool UseSeparateMainExecutable(BuildOptions buildOptions)
         {
-            return UseSymbolsExports && OutputType == TargetOutputType.Executable && !buildOptions.Platform.HasExecutableFileReferenceSupport && !Configuration.BuildBindingsOnly;
+            if (OutputType == TargetOutputType.Executable && !Configuration.BuildBindingsOnly)
+            {
+                return !IsMonolithicExecutable || (!buildOptions.Platform.HasExecutableFileReferenceSupport && UseSymbolsExports);
+            }
+            return false;
         }
 
         private void BuildMainExecutable(TaskGraph graph, BuildOptions buildOptions)
@@ -175,7 +198,10 @@ namespace Flax.Build
             Builder.BuildModuleInner(buildData, mainModule, mainModuleOptions, false);
 
             // Link executable
-            exeBuildOptions.LinkEnv.InputLibraries.Add(Path.Combine(buildOptions.OutputFolder, buildOptions.Platform.GetLinkOutputFileName(OutputName, LinkerOutput.SharedLibrary)));
+            var engineLibraryType = LinkerOutput.SharedLibrary;
+            if (buildOptions.Toolchain?.Compiler == TargetCompiler.MSVC)
+                engineLibraryType = LinkerOutput.ImportLibrary; // MSVC links DLL against import library
+            exeBuildOptions.LinkEnv.InputLibraries.Add(Path.Combine(buildOptions.OutputFolder, buildOptions.Platform.GetLinkOutputFileName(LibraryName, engineLibraryType)));
             exeBuildOptions.LinkEnv.InputFiles.AddRange(mainModuleOptions.OutputFiles);
             exeBuildOptions.DependencyFiles.AddRange(mainModuleOptions.DependencyFiles);
             exeBuildOptions.OptionalDependencyFiles.AddRange(mainModuleOptions.OptionalDependencyFiles);