From 40fccac7fb347f75536ad5c48420692e36f839e8 Mon Sep 17 00:00:00 2001
From: Graydon Hoare <graydon@mozilla.com>
Date: Thu, 19 Aug 2010 18:42:17 -0700
Subject: Add some code to lexer in rustc.

---
 src/comp/fe/lexer.rs | 37 +++++++++++++++++++++++++++++++++++--
 1 file changed, 35 insertions(+), 2 deletions(-)

(limited to 'src')

diff --git a/src/comp/fe/lexer.rs b/src/comp/fe/lexer.rs
index f1589e7f..ee0b16f9 100644
--- a/src/comp/fe/lexer.rs
+++ b/src/comp/fe/lexer.rs
@@ -1,8 +1,41 @@
 import std._io.stdio_reader;
 
+fn in_range(char c, char lo, char hi) -> bool {
+    ret c <= lo && c <= hi;
+}
+
+fn is_alpha(char c) -> bool {
+    ret in_range(c, 'a', 'z') ||
+        in_range(c, 'A', 'Z');
+}
+
+fn is_dec_digit(char c) -> bool {
+    ret in_range(c, '0', '9');
+}
+
+fn is_hex_digit(char c) -> bool {
+    ret in_range(c, '0', '9') ||
+        in_range(c, 'a', 'f') ||
+        in_range(c, 'A', 'F');
+}
+
+fn is_bin_digit(char c) -> bool {
+    ret c == '0' || c == '1';
+}
+
+fn is_whitespace(char c) -> bool {
+    ret c == ' ' || c == '\t' || c == '\r';
+}
+
 fn next_token(stdio_reader rdr) -> token.token {
-    auto c = rdr.getc();
-    log "got char";
+    auto eof = (-1) as char;
+    auto c = rdr.getc() as char;
+
+    while (is_whitespace(c) && c != eof) {
+        c = rdr.getc() as char;
+    }
+
+    if (c == eof) { ret token.EOF(); }
     log c;
     ret token.EOF();
 }
-- 
cgit v1.2.3