// Copyright (c) 2012-2023 Wojciech Figat. All rights reserved. #include "TextProcessing.h" TextProcessing::TextProcessing(const char* input, int32 length) : _buffer(input) , _length(length) , _cursor(const_cast(input)) , _position(0) , _line(1) , Separators(32) , Whitespaces(8) { } void TextProcessing::Setup_HLSL() { static SeparatorData separators[] = { { '\r', '\n' }, { '/', '/' }, { '/', '*' }, { '\n', 0 }, { '\t', 0 }, { ' ', 0 }, { '.', 0 }, { ',', 0 }, { ':', 0 }, { ';', 0 }, { '+', 0 }, { '-', 0 }, { '(', 0 }, { ')', 0 }, { '!', 0 }, { '=', 0 }, { '&', 0 }, { '%', 0 }, { '*', 0 }, { '<', 0 }, { '>', 0 }, { '[', 0 }, { ']', 0 }, { '{', 0 }, { '}', 0 }, }; Separators.Set(separators, ARRAY_COUNT(separators)); static char whitespaces[] = { 9, 10, 11, 12, 13, 32, }; Whitespaces.Set(whitespaces, ARRAY_COUNT(whitespaces)); } char TextProcessing::ReadChar() { if (_position >= _length) return 0; return moveForward(); } void TextProcessing::EatWhiteSpaces() { // Read until can while (_position < _length) { // Peek char char c = PeekChar(); // Check if is whitespace bool isToken = true; for (int32 i = 0; i < Whitespaces.Count(); i++) { if (Whitespaces[i] == c) { // Read it moveForward(); isToken = false; break; } } if (isToken) { // End return; } } } void TextProcessing::ReadToken(Token* token) { // Eat whitespaces EatWhiteSpaces(); // Set token start but reset its length token->Start = _cursor; token->Length = 0; token->Separator = { 0, 0 }; // Read until can while (_position < _length) { // Read char char c = moveForward(); // Check if it's a separator char char c1 = PeekChar(); for (int32 s = 0; s < Separators.Count(); s++) { if (Separators[s].C0 == c && Separators[s].C1 == c1) { /*if (*length == 0) { // Skip separator after separator case and start searching for new token *token = _cursor; *length = -1; break; }*/ token->Separator = Separators[s]; ReadChar(); return; } } for (int32 s = 0; s < Separators.Count(); s++) { if (Separators[s].C0 == c && Separators[s].C1 == 0) { if (token->Length == 0) { // Skip separator after separator case and start searching for new token token->Start = _cursor; token->Length = -1; break; } token->Separator = Separators[s]; return; } } // Move to the next character token->Length++; } } void TextProcessing::ReadLine(Token* token) { // Set line start but reset its length token->Start = _cursor; token->Length = 0; // Read until can while (_position < _length) { // Read char char c = moveForward(); // Check new-line char if (c == '\n') { return; } // Move to the next char token->Length++; } } void TextProcessing::ReadLine() { // Read until can while (_position < _length) { // Read char char c = moveForward(); // Check new-line char if (c == '\n') { return; } } } char TextProcessing::moveForward() { char c = *_cursor; if (c == '\n') _line++; _cursor++; _position++; return c; } char TextProcessing::moveBack() { _position--; _cursor--; char c = *_cursor; if (c == '\n') _line--; return c; }