kotofyt/funnygame
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

simple B compiler

Browse Source
This commit is contained in:
kotofyt
2026-01-17 11:24:00 +02:00
parent c69f589439
commit ee7735b610
7 changed files with 152 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
tier2/tokenizer.cpp
View File

@@ -18,13 +18,14 @@ CUtlVector<Token_t> Tokenize( const char *psz )
return Tokenize(psz, IsWordSymbol);
}
CUtlVector<Token_t> Tokenize( const char *psz, fnIsAlphabetSymbol fnIsAlphabetSymbol )
CUtlVector<Token_t> Tokenize( const char *psz, IsAlphabetSymbolFn fnIsAlphabetSymbol )
{
CUtlVector<Token_t> tokens = {};
size_t i = 0;
char c;
uint32_t nCurrentLine = 0;
uint32_t nCurrentLine = 1;
uint32_t nCurrentCharacter = 0;
uint32_t nStartingCharacter = 0;
bool bIsQuoted = false;
bool bIsSlash = false;
CUtlString szStringValue;
@@ -32,14 +33,17 @@ CUtlVector<Token_t> Tokenize( const char *psz, fnIsAlphabetSymbol fnIsAlphabetSy
while (true)
{
c = psz[i];
i++;
if (c == '\0')
break;
if (c == '\n')
{
nCurrentCharacter = 0;
nStartingCharacter = 0;
nCurrentLine++;
}
else
nCurrentCharacter++;
i++;
if (bIsQuoted)
@@ -91,7 +95,7 @@ CUtlVector<Token_t> Tokenize( const char *psz, fnIsAlphabetSymbol fnIsAlphabetSy
bIsQuoted = false;
if (szStringValue == 0)
continue;
tokens.AppendTail({szStringValue, true, nCurrentLine});
tokens.AppendTail({szStringValue, true, nCurrentLine, nStartingCharacter});
szStringValue = 0;
continue;
default:
@@ -113,10 +117,12 @@ CUtlVector<Token_t> Tokenize( const char *psz, fnIsAlphabetSymbol fnIsAlphabetSy
continue;
} else {
if (szStringValue != 0)
tokens.AppendTail({szStringValue, false, nCurrentLine});
tokens.AppendTail({szStringValue, false, nCurrentLine, nStartingCharacter});
nStartingCharacter = nCurrentCharacter;
szStringValue = 0;
if (V_isgraph(c) && c != '"')
tokens.AppendTail({CUtlString("%c", c), false, nCurrentLine});
tokens.AppendTail({CUtlString("%c", c), false, nCurrentLine, nStartingCharacter});
}
}
};