1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
|
using System.Reflection;
using System.Text.RegularExpressions;
namespace AtLangCompiler;
internal class Lexer
{
private readonly string text;
public IReadOnlyDictionary<string, TokenType> methodTokenTypeMap;
private int pos = 0;
public Lexer(string text)
{
this.text = text;
this.methodTokenTypeMap = Assembly.GetExecutingAssembly()
.GetTypes()
.Where(t => typeof(ILexerTokenConfig).IsAssignableFrom(t) && !t.IsAbstract && !t.IsInterface)
.Select(configType => (ILexerTokenConfig)Activator.CreateInstance(configType)!)
.SelectMany(config => config.TokenStrings)
.ToDictionary(pair => pair.Key, pair => pair.Value);
}
private char Current => pos < text.Length ? text[pos] : '\0';
public Token NextToken()
{
// Skip whitespace
while (pos < text.Length && char.IsWhiteSpace(Current))
{
pos++;
}
if (pos >= text.Length)
{
return new Token(TokenType.EOF, string.Empty);
}
string rest = text.Substring(pos);
foreach (KeyValuePair<string, TokenType> methodToken in methodTokenTypeMap)
{
if (rest.StartsWith(methodToken.Key))
{
pos += methodToken.Key.Length;
return new Token(methodToken.Value, methodToken.Key);
}
}
// ==
if (pos + 1 < text.Length && Current == '=' && text[pos + 1] == '=')
{
pos += 2;
return new Token(TokenType.EQEQ, "==");
}
// single-char tokens
switch (Current)
{
case '@':
pos++;
return new Token(TokenType.AT, "@");
case '=':
pos++;
return new Token(TokenType.EQUAL, "=");
case '+':
pos++;
return new Token(TokenType.PLUS, "+");
case '(':
pos++;
return new Token(TokenType.LPAREN, "(");
case ')':
pos++;
return new Token(TokenType.RPAREN, ")");
case '{':
pos++;
return new Token(TokenType.LBRACE, "{");
case '}':
pos++;
return new Token(TokenType.RBRACE, "}");
case '"':
pos++;
int start = pos;
while (pos < text.Length && text[pos] != '"') pos++;
string strVal = text.Substring(start, pos - start);
if (pos < text.Length) pos++; // skip closing quote
return new Token(TokenType.STRING, strVal);
}
// identifier?
if (Regex.IsMatch(Current.ToString(), "[A-Za-z_]"))
{
int startPos = pos;
while (pos < text.Length &&
Regex.IsMatch(text[pos].ToString(), "[A-Za-z0-9_]"))
{
pos++;
}
string ident = text.Substring(startPos, pos - startPos);
return new Token(TokenType.IDENT, ident);
}
// skip unrecognized
pos++;
return NextToken();
}
}
|
