Fix shaders parsing to skip comments in between special macros

Source/Engine/Utilities/TextProcessing.cpp

@@ -1,6 +1,7 @@
 // Copyright (c) 2012-2024 Wojciech Figat. All rights reserved.
 
 #include "TextProcessing.h"
+#include "Engine/Core/Log.h"
 
 TextProcessing::TextProcessing(const char* input, int32 length)
     : _buffer(input)
@@ -54,6 +55,8 @@ void TextProcessing::Setup_HLSL()
         32,
     };
     Whitespaces.Set(whitespaces, ARRAY_COUNT(whitespaces));
+    SingleLineComment = SeparatorData('/', '/');
+    MultiLineCommentSeparator = SeparatorData('/', '*');
 }
 
 char TextProcessing::ReadChar()
@@ -123,6 +126,38 @@ void TextProcessing::ReadToken(Token* token)
 
                 token->Separator = Separators[s];
                 ReadChar();
+                
+                // Check for comments
+                if (token->Separator == SingleLineComment)
+                {
+                    // Read whole line
+                    ReadLine();
+
+                    // Read another token
+                    ReadToken(token);
+                }
+                else if (token->Separator == MultiLineCommentSeparator)
+                {
+                    // Read tokens until end sequence
+                    char prev = ' ';
+                    while (CanRead())
+                    {
+                        c = ReadChar();
+                        if (prev == '*' && c == '/')
+                            break;
+                        prev = c;
+                    }
+
+                    // Check if comment is valid (has end before file end)
+                    if (!CanRead())
+                    {
+                        LOG(Warning, "Missing multiline comment ending");
+                    }
+
+                    // Read another token
+                    ReadToken(token);
+                }
+
                 return;
             }
         }

Source/Engine/Utilities/TextProcessing.h

@@ -9,13 +9,13 @@
 #include "Engine/Platform/StringUtils.h"
 
 /// <summary>
-/// Helper class to fast ANSI text processing (tokenization, reading, streaming etc.)
+/// Helper class to fast ANSI text processing (tokenization, reading, streaming etc.).
 /// </summary>
 class FLAXENGINE_API TextProcessing : public NonCopyable
 {
 public:
     /// <summary>
-    /// Separator structure
+    /// Separator structure.
     /// </summary>
     struct SeparatorData
     {
@@ -200,18 +200,21 @@ public:
 
 public:
     /// <summary>
-    /// Array with all token separators
+    /// Array with all token separators.
     /// </summary>
     Array<SeparatorData> Separators;
 
     /// /// <summary>
-    /// Array with all white characters
+    /// Array with all white characters.
     /// </summary>
     Array<char> Whitespaces;
 
+    SeparatorData SingleLineComment;
+    SeparatorData MultiLineCommentSeparator;
+
 public:
     /// <summary>
-    /// Set separators and white chars for HLSL language
+    /// Sets up separators and white chars for HLSL language.
     /// </summary>
     void Setup_HLSL();
 
@@ -219,25 +222,22 @@ public:
     /// <summary>
     /// Returns true if there are still characters in the buffer and can read data from it
     /// </summary>
-    /// <returns>True if can read data, otherwise false</returns>
     FORCE_INLINE bool CanRead() const
     {
         return _position < _length;
     }
 
     /// <summary>
-    /// Peeks single character without moving forward in the buffer
+    /// Peeks a single character without moving forward in the buffer.
     /// </summary>
-    /// <returns>First character</returns>
     FORCE_INLINE char PeekChar() const
     {
         return *_cursor;
     }
 
     /// <summary>
-    /// Gets current line number
+    /// Gets the current line number.
     /// </summary>
-    /// <returns>Current line number</returns>
     FORCE_INLINE int32 GetLine() const
     {
         return _line;