Lexer: include leading whitespace in the comment token.

Indented comments are ignored rather than causing errors.

1 files changed, 63 insertions, 53 deletions

diff --git a/src/lexer.cc b/src/lexer.cc
index 75b91e7..6bcdedc 100644
--- a/src/lexer.cc
+++ b/src/lexer.cc
@@ -104,39 +104,40 @@ Lexer::Token Lexer::ReadToken() {
     
 {
 	char yych;
+	unsigned int yyaccept = 0;
 	static const unsigned char yybm[] = {
-		  0, 128, 128, 128, 128, 128, 128, 128, 
-		128, 128,   0, 128, 128, 128, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
-		192, 128, 128, 128, 128, 128, 128, 128, 
-		128, 128, 128, 128, 128, 160, 160, 128, 
-		160, 160, 160, 160, 160, 160, 160, 160, 
-		160, 160, 128, 128, 128, 128, 128, 128, 
-		128, 160, 160, 160, 160, 160, 160, 160, 
-		160, 160, 160, 160, 160, 160, 160, 160, 
-		160, 160, 160, 160, 160, 160, 160, 160, 
-		160, 160, 160, 128, 128, 128, 128, 160, 
-		128, 160, 160, 160, 160, 160, 160, 160, 
-		160, 160, 160, 160, 160, 160, 160, 160, 
-		160, 160, 160, 160, 160, 160, 160, 160, 
-		160, 160, 160, 128, 128, 128, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
-		128, 128, 128, 128, 128, 128, 128, 128, 
+		  0,  64,  64,  64,  64,  64,  64,  64, 
+		 64,  64,   0,  64,  64,  64,  64,  64, 
+		 64,  64,  64,  64,  64,  64,  64,  64, 
+		 64,  64,  64,  64,  64,  64,  64,  64, 
+		192,  64,  64,  64,  64,  64,  64,  64, 
+		 64,  64,  64,  64,  64,  96,  96,  64, 
+		 96,  96,  96,  96,  96,  96,  96,  96, 
+		 96,  96,  64,  64,  64,  64,  64,  64, 
+		 64,  96,  96,  96,  96,  96,  96,  96, 
+		 96,  96,  96,  96,  96,  96,  96,  96, 
+		 96,  96,  96,  96,  96,  96,  96,  96, 
+		 96,  96,  96,  64,  64,  64,  64,  96, 
+		 64,  96,  96,  96,  96,  96,  96,  96, 
+		 96,  96,  96,  96,  96,  96,  96,  96, 
+		 96,  96,  96,  96,  96,  96,  96,  96, 
+		 96,  96,  96,  64,  64,  64,  64,  64, 
+		 64,  64,  64,  64,  64,  64,  64,  64, 
+		 64,  64,  64,  64,  64,  64,  64,  64, 
+		 64,  64,  64,  64,  64,  64,  64,  64, 
+		 64,  64,  64,  64,  64,  64,  64,  64, 
+		 64,  64,  64,  64,  64,  64,  64,  64, 
+		 64,  64,  64,  64,  64,  64,  64,  64, 
+		 64,  64,  64,  64,  64,  64,  64,  64, 
+		 64,  64,  64,  64,  64,  64,  64,  64, 
+		 64,  64,  64,  64,  64,  64,  64,  64, 
+		 64,  64,  64,  64,  64,  64,  64,  64, 
+		 64,  64,  64,  64,  64,  64,  64,  64, 
+		 64,  64,  64,  64,  64,  64,  64,  64, 
+		 64,  64,  64,  64,  64,  64,  64,  64, 
+		 64,  64,  64,  64,  64,  64,  64,  64, 
+		 64,  64,  64,  64,  64,  64,  64,  64, 
+		 64,  64,  64,  64,  64,  64,  64,  64, 
 	};
 
 	yych = *p;
@@ -144,11 +145,12 @@ Lexer::Token Lexer::ReadToken() {
 		if (yych <= ',') {
 			if (yych <= 0x1F) {
 				if (yych <= 0x00) goto yy21;
-				if (yych == '\n') goto yy4;
+				if (yych == '\n') goto yy6;
 				goto yy23;
 			} else {
-				if (yych <= ' ') goto yy6;
-				if (yych != '#') goto yy23;
+				if (yych <= ' ') goto yy2;
+				if (yych == '#') goto yy4;
+				goto yy23;
 			}
 		} else {
 			if (yych <= ':') {
@@ -185,19 +187,21 @@ Lexer::Token Lexer::ReadToken() {
 			}
 		}
 	}
+yy2:
+	yyaccept = 0;
 	yych = *(q = ++p);
-	if (yych >= 0x01) goto yy62;
+	goto yy65;
 yy3:
-	{ token = ERROR;    break; }
+	{ token = INDENT;   break; }
 yy4:
-	++p;
-	{ token = NEWLINE;  break; }
+	yyaccept = 1;
+	yych = *(q = ++p);
+	if (yych >= 0x01) goto yy60;
+yy5:
+	{ token = ERROR;    break; }
 yy6:
 	++p;
-	yych = *p;
-	goto yy60;
-yy7:
-	{ token = INDENT;   break; }
+	{ token = NEWLINE;  break; }
 yy8:
 	++p;
 	if ((yych = *p) == 'u') goto yy54;
@@ -238,7 +242,7 @@ yy21:
 	{ token = TEOF;     break; }
 yy23:
 	yych = *++p;
-	goto yy3;
+	goto yy5;
 yy24:
 	++p;
 	yych = *p;
@@ -329,20 +333,26 @@ yy60:
 	if (yybm[0+yych] & 64) {
 		goto yy59;
 	}
-	goto yy7;
-yy61:
+	if (yych >= 0x01) goto yy62;
+	p = q;
+	if (yyaccept <= 0) {
+		goto yy3;
+	} else {
+		goto yy5;
+	}
+yy62:
 	++p;
+	{ continue; }
+yy64:
+	yyaccept = 0;
+	q = ++p;
 	yych = *p;
-yy62:
+yy65:
 	if (yybm[0+yych] & 128) {
-		goto yy61;
+		goto yy64;
 	}
-	if (yych >= 0x01) goto yy64;
-	p = q;
+	if (yych == '#') goto yy59;
 	goto yy3;
-yy64:
-	++p;
-	{ continue; }
 }
 
   }