Extended the lexer and added a test.

.gitignore

@@ -1,8 +1,10 @@
 .cache/
 .idea/
 build/
+builddir/
 # Ignore all subdirectories (and their contents) under 'subprojects/'
 subprojects/*/
 # Optional: Ignore Meson's internal download tracking file (if unwanted)
 subprojects/*.cache
 subprojects/packagecache
+.vscode/

src/faust-lib/parser/faustLexer.cpp

@@ -18,12 +18,20 @@ FaustLexer::FaustLexer() : lexer(TK_EOF) {
   lexer.addToken(u"->", TK_ARROW);
   lexer.addToken(u":", TK_COLON);
   lexer.addToken(u";", TK_SEMICOLON);
+  lexer.addToken(u"\\*", TK_ASTERISK);
   lexer.addToken(u"\\/\\/[^\\r\\n]*", TK_COMMENT);
   lexer.addToken(u"\\/\\*[\\s\\S]*?\\*\\/", TK_MULTILINE_COMMENT);
   lexer.addToken(u"\\bfn\\b", TK_FN);
   lexer.addToken(u"\\bconst\\b", TK_CONST);
   lexer.addToken(u"\\bmut\\b", TK_MUT);
   lexer.addToken(u"\\bimport\\b", TK_IMPORT);
+  lexer.addToken(u"\\bfrom\\b", TK_FROM);
+  lexer.addToken(u"\\bexternal\\b", TK_EXTERNAL);
+  lexer.addToken(u"\\bcheck\\b", TK_CHECK);
+  lexer.addToken(u"\\bforce\\b", TK_FORCE);
+  lexer.addToken(u"\\bwhile\\b", TK_WHILE);
+  lexer.addToken(u"\\bif\\b", TK_IF);
+  lexer.addToken(u"\\belse\\b", TK_ELSE);
   lexer.addToken(u"\\b[a-zA-Z][a-zA-Z0-9_]*\\b", TK_IDENTIFIER);
 }
 

src/faust-lib/parser/faustLexer.h

@@ -23,12 +23,20 @@ enum FaustTokens {
   TK_ARROW,
   TK_COLON,
   TK_SEMICOLON,
+  TK_ASTERISK,
   TK_COMMENT,
   TK_MULTILINE_COMMENT,
   TK_FN,
   TK_CONST,
   TK_MUT,
   TK_IMPORT,
+  TK_FROM,
+  TK_EXTERNAL,
+  TK_WHILE,
+  TK_IF,
+  TK_ELSE,
+  TK_CHECK,
+  TK_FORCE,
   TK_IDENTIFIER,
 };
 

tests/mainTest.cpp

@@ -0,0 +1,11 @@
+#include <gtest/gtest.h>
+
+#include "faust-lib/parser/faustLexer.h"
+
+// Simple test case without fixture
+TEST(TokenTest, NoError) {
+  icu::UnicodeString text = "import \"test\";";
+  faust::FaustLexer lexer;
+  auto lexemes = lexer.lex(text);
+  EXPECT_EQ(lexemes[0].token, faust::TK_IMPORT);
+}

tests/meson.build

@@ -1,3 +1,5 @@
 test_exe = executable('test_gtest', 'mainTest.cpp',
-                      dependencies: [gmock_main_dep, gtest_dep, gmock_dep])
+                      dependencies: [gmock_main_dep, gtest_dep, gmock_dep, icu_dep, icu_i18_dep, icu_data_dep],
+                      link_with: faust_parser_lib,
+                      include_directories: faust_src_root_include)
 test('GTest Suite', test_exe)