Fix shaders parsing to skip comments in between special macros

Source/Engine/ShadersCompilation/Parser/ShaderProcessing.cpp

@@ -66,10 +66,6 @@ void ShaderProcessing::Parser::init()
 bool ShaderProcessing::Parser::process()
 {
     const Token defineToken("#define");
-    const Separator singleLineCommentSeparator('/', '/');
-    const Separator multiLineCommentSeparator('/', '*');
-
-    // TODO: split parsing into two phrases: comments preprocessing and parsing
 
     // Read whole source code
     Token token;
@@ -77,36 +73,8 @@ bool ShaderProcessing::Parser::process()
     {
         text.ReadToken(&token);
 
-        // Single line comment
-        if (token.Separator == singleLineCommentSeparator)
-        {
-            // Read whole line
-            text.ReadLine();
-        }
-            // Multi line comment
-        else if (token.Separator == multiLineCommentSeparator)
-        {
-            // Read tokens until end sequence
-            char prev = ' ';
-            char c;
-            while (text.CanRead())
-            {
-                c = text.ReadChar();
-                if (prev == '*' && c == '/')
-                {
-                    break;
-                }
-                prev = c;
-            }
-
-            // Check if comment is valid (has end before file end)
-            if (!text.CanRead())
-            {
-                OnWarning(TEXT("Missing multiline comment ending"));
-            }
-        }
-            // Preprocessor definition
-        else if (token == defineToken)
+        // Preprocessor definition
+        if (token == defineToken)
         {
             // Skip
             text.ReadLine();

Source/Engine/Utilities/TextProcessing.cpp

@@ -1,6 +1,7 @@
 // Copyright (c) 2012-2024 Wojciech Figat. All rights reserved.
 
 #include "TextProcessing.h"
+#include "Engine/Core/Log.h"
 
 TextProcessing::TextProcessing(const char* input, int32 length)
     : _buffer(input)
@@ -54,6 +55,8 @@ void TextProcessing::Setup_HLSL()
         32,
     };
     Whitespaces.Set(whitespaces, ARRAY_COUNT(whitespaces));
+    SingleLineComment = SeparatorData('/', '/');
+    MultiLineCommentSeparator = SeparatorData('/', '*');
 }
 
 char TextProcessing::ReadChar()
@@ -123,6 +126,38 @@ void TextProcessing::ReadToken(Token* token)
 
                 token->Separator = Separators[s];
                 ReadChar();
+                
+                // Check for comments
+                if (token->Separator == SingleLineComment)
+                {
+                    // Read whole line
+                    ReadLine();
+
+                    // Read another token
+                    ReadToken(token);
+                }
+                else if (token->Separator == MultiLineCommentSeparator)
+                {
+                    // Read tokens until end sequence
+                    char prev = ' ';
+                    while (CanRead())
+                    {
+                        c = ReadChar();
+                        if (prev == '*' && c == '/')
+                            break;
+                        prev = c;
+                    }
+
+                    // Check if comment is valid (has end before file end)
+                    if (!CanRead())
+                    {
+                        LOG(Warning, "Missing multiline comment ending");
+                    }
+
+                    // Read another token
+                    ReadToken(token);
+                }
+
                 return;
             }
         }

Source/Engine/Utilities/TextProcessing.h

@@ -9,13 +9,13 @@
 #include "Engine/Platform/StringUtils.h"
 
 /// <summary>
-/// Helper class to fast ANSI text processing (tokenization, reading, streaming etc.)
+/// Helper class to fast ANSI text processing (tokenization, reading, streaming etc.).
 /// </summary>
 class FLAXENGINE_API TextProcessing : public NonCopyable
 {
 public:
     /// <summary>
-    /// Separator structure
+    /// Separator structure.
     /// </summary>
     struct SeparatorData
     {
@@ -200,18 +200,21 @@ public:
 
 public:
     /// <summary>
-    /// Array with all token separators
+    /// Array with all token separators.
     /// </summary>
     Array<SeparatorData> Separators;
 
     /// /// <summary>
-    /// Array with all white characters
+    /// Array with all white characters.
     /// </summary>
     Array<char> Whitespaces;
 
+    SeparatorData SingleLineComment;
+    SeparatorData MultiLineCommentSeparator;
+
 public:
     /// <summary>
-    /// Set separators and white chars for HLSL language
+    /// Sets up separators and white chars for HLSL language.
     /// </summary>
     void Setup_HLSL();
 
@@ -219,25 +222,22 @@ public:
     /// <summary>
     /// Returns true if there are still characters in the buffer and can read data from it
     /// </summary>
-    /// <returns>True if can read data, otherwise false</returns>
     FORCE_INLINE bool CanRead() const
     {
         return _position < _length;
     }
 
     /// <summary>
-    /// Peeks single character without moving forward in the buffer
+    /// Peeks a single character without moving forward in the buffer.
     /// </summary>
-    /// <returns>First character</returns>
     FORCE_INLINE char PeekChar() const
     {
         return *_cursor;
     }
 
     /// <summary>
-    /// Gets current line number
+    /// Gets the current line number.
     /// </summary>
-    /// <returns>Current line number</returns>
     FORCE_INLINE int32 GetLine() const
     {
         return _line;