diff options
| author | Marijn Haverbeke <[email protected]> | 2011-03-24 16:07:43 +0100 |
|---|---|---|
| committer | Graydon Hoare <[email protected]> | 2011-03-25 08:23:00 -0700 |
| commit | d4c3059e1ef6dfe0796cb9badd0773ab21d96c42 (patch) | |
| tree | 9b243aeff79918f2f7d0a32773688a59e6f6307d /src | |
| parent | Start making the standard-lib utf-8 aware (diff) | |
| download | rust-d4c3059e1ef6dfe0796cb9badd0773ab21d96c42.tar.xz rust-d4c3059e1ef6dfe0796cb9badd0773ab21d96c42.zip | |
make lexer unicode-aware for strings and char literals
Diffstat (limited to 'src')
| -rw-r--r-- | src/comp/front/lexer.rs | 10 | ||||
| -rw-r--r-- | src/comp/front/token.rs | 4 | ||||
| -rw-r--r-- | src/comp/pretty/pprust.rs | 1 |
3 files changed, 7 insertions, 8 deletions
diff --git a/src/comp/front/lexer.rs b/src/comp/front/lexer.rs index 9c05b570..791276a7 100644 --- a/src/comp/front/lexer.rs +++ b/src/comp/front/lexer.rs @@ -76,7 +76,7 @@ impure fn new_reader(io.reader rdr, str filename) -> reader col += 1u; } - n = rdr.read_byte() as char; + n = rdr.read_char(); } fn mark() { @@ -204,8 +204,8 @@ impure fn new_reader(io.reader rdr, str filename) -> reader reserved.insert("m128", ()); // IEEE 754-2008 'decimal128' reserved.insert("dec", ()); // One of m32, m64, m128 - ret reader(rdr, filename, rdr.read_byte() as char, - rdr.read_byte() as char, 1u, 0u, 1u, 0u, keywords, reserved); + ret reader(rdr, filename, rdr.read_char(), + rdr.read_char(), 1u, 0u, 1u, 0u, keywords, reserved); } @@ -505,7 +505,7 @@ impure fn next_token(reader rdr) -> token.token { if (is_alpha(c) || c == '_') { while (is_alnum(c) || c == '_') { - _str.push_byte(accum_str, (c as u8)); + _str.push_char(accum_str, c); rdr.bump(); c = rdr.curr(); } @@ -692,7 +692,7 @@ impure fn next_token(reader rdr) -> token.token { } } case (_) { - _str.push_byte(accum_str, rdr.curr() as u8); + _str.push_char(accum_str, rdr.curr()); } } rdr.bump(); diff --git a/src/comp/front/token.rs b/src/comp/front/token.rs index 46fd0735..72d82010 100644 --- a/src/comp/front/token.rs +++ b/src/comp/front/token.rs @@ -303,9 +303,9 @@ fn to_str(token t) -> str { ret "\"" + s + "\""; } case (LIT_CHAR(?c)) { - // FIXME: escape and encode. + // FIXME: escape. auto tmp = "'"; - _str.push_byte(tmp, c as u8); + _str.push_char(tmp, c); _str.push_byte(tmp, '\'' as u8); ret tmp; } diff --git a/src/comp/pretty/pprust.rs b/src/comp/pretty/pprust.rs index 0e30ced1..25eb60ed 100644 --- a/src/comp/pretty/pprust.rs +++ b/src/comp/pretty/pprust.rs @@ -718,7 +718,6 @@ impure fn print_maybe_parens(ps s, @ast.expr expr, int outer_prec) { if (add_them) {pclose(s);} } -// TODO non-ascii fn escape_str(str st, char to_escape) -> str { let str out = ""; auto len = _str.byte_len(st); |