From: Nobuyoshi Nakada <nobu@...>
Date: 2007-10-25T17:05:21+09:00
Subject: Re: \u escapes in string literals: proof of concept  implementation

Hi,

At Thu, 25 Oct 2007 15:30:39 +0900,
David Flanagan wrote in [ruby-core:12923]:
> Well if you're writing the patch, we can take the "proof of concept" out 
> of the subject line!

But it didn't work enough.

> Thanks for this!  My recent patch (just posted a bit before yours) 
> includes a version of Martin's test cases in 
> tests/ruby/test_unicode_escape.rb

Thank you, I ran it and found a few bugs, and also found
oniguruma doesn't take care of \u by default.  It may be
reasonable to convert them in the parser, but /\x41/ == /A/ has
returned false...

An additional patch to the previous one.  With this patch,
\x{...} means a codepoint in the given encoding, while \xXX
means a raw byte as well as it's done now.


diff -U2 parse.y parse.y
--- parse.y	(working copy)
+++ parse.y	(working copy)
@@ -4952,4 +4952,5 @@
 	c = scan_hex(lex_p, 8, numlen);
 	if (!*numlen) goto invalid;
+	lex_p += *numlen;
 	if (!peek('}')) {
 	    yyerror("unterminated hex escape");
@@ -4966,4 +4967,5 @@
 	    return 0;
 	}
+	lex_p += *numlen;
     }
     return c;
@@ -4982,5 +4984,5 @@
 	    return 0;
 	}
-	if (codepoint > 0x7fffffff) {
+	if (codepoint > 0x10ffff) {
 	    yyerror("illegal Unicode codepoint (too large)");
 	    return 0;
@@ -5015,5 +5017,7 @@
 {
     int c;
+    int numlen;
 
+    if (mb) *mb = ENC_CODERANGE_SINGLE;
     switch (c = nextc()) {
       case '\\':	/* Backslash */
@@ -5056,22 +5060,19 @@
       case 'x':	/* hex constant */
 	if (flags & (ESCAPE_CONTROL|ESCAPE_META)) goto eof;
-	{
-	    int numlen;
-
-	    c = tok_hex(&numlen);
-	    if (numlen == 0) goto eof;
+	c = tok_hex(&numlen);
+	if (numlen == 0) return 0;
+	if (mb && (c >= 0x80)) {
+	    if (numlen == 2)
+		*mb = ENC_CODERANGE_UNKNOWN;
+	    else
+		*mb = ENC_CODERANGE_MULTI;
 	}
-	if (mb && (c >= 0x80)) *mb = ENC_CODERANGE_UNKNOWN;
 	return c;
 
       case 'u':	/* hex constant */
 	if (flags & (ESCAPE_CONTROL|ESCAPE_META)) goto eof;
-	{
-	    int numlen;
-
-	    c = tok_utf8(&numlen, encp);
-	    if (numlen == 0) goto eof;
-	}
-	if (mb && (c >= 0x80)) *mb = ENC_CODERANGE_UNKNOWN;
+	c = tok_utf8(&numlen, encp);
+	if (numlen == 0) return 0;
+	if (mb && (c >= 0x80)) *mb = ENC_CODERANGE_MULTI;
 	return c;
 
@@ -5125,4 +5126,13 @@
 #define tokcopy(n) memcpy(tokspace(n), lex_p - (n), (n))
 
+static void
+parser_tokaddmbc(struct parser_params *parser, int c, rb_encoding *enc)
+{
+    int len = rb_enc_codelen(c, enc);
+    rb_enc_mbcput(c, tokspace(len), enc);
+}
+
+#define tokaddmbc(c, enc) parser_tokaddmbc(parser, c, enc)
+
 static int
 parser_tokadd_escape(struct parser_params *parser, int term, int *mb, rb_encoding **encp)
@@ -5173,8 +5183,6 @@
 	    uc = tok_utf8(&numlen, encp);
 	    if (numlen == 0) goto eof;
-	    lex_p += numlen;
-	    tokcopy(numlen + 2);
+	    tokaddmbc(uc, *encp);
 	    if (mb && (uc >= 0x80)) *mb = ENC_CODERANGE_MULTI;
-	    if (uc >= 0x80) return 1;
 	}
 	return 0;
@@ -5294,13 +5302,4 @@
 #define tokadd_mbchar(c) parser_tokadd_mbchar(parser, c)
 
-static void
-parser_tokaddmbc(struct parser_params *parser, int c, rb_encoding *enc)
-{
-    int len = rb_enc_codelen(c, enc);
-    rb_enc_mbcput(c, tokspace(len), enc);
-}
-
-#define tokaddmbc(c, enc) parser_tokaddmbc(parser, c, enc)
-
 static int
 parser_tokadd_string(struct parser_params *parser,
@@ -5374,13 +5373,16 @@
 		}
 		else if (func & STR_FUNC_EXPAND) {
+		    int tmb = ENC_CODERANGE_SINGLE;
 		    pushback(c);
 		    if (func & STR_FUNC_ESCAPE) tokadd('\\');
-		    c = read_escape(0, mb, &enc);
-		    if (has_mb && enc != *encp) {
-			mixed_escape(beg, enc, *encp);
-			continue;
-		    }
-		    if (c >= 0x80) {
-			tokaddmbc(c, enc);
+		    c = read_escape(0, &tmb, &enc);
+		    if (mb) *mb = tmb;
+		    if (tmb == ENC_CODERANGE_MULTI) {
+			if (has_mb && enc != *encp) {
+			    mixed_escape(beg, enc, *encp);
+			}
+			else {
+			    tokaddmbc(c, enc);
+			}
 			continue;
 		    }
@@ -6256,7 +6258,11 @@
 	    goto ternary;
 	}
-	else if (c == '\\' && (c = read_escape(0, 0, &enc)) >= 0x80) {
-	    mb = ENC_CODERANGE_UNKNOWN;
-	    tokaddmbc(c, enc);
+	else if (c == '\\' && (c = read_escape(0, &mb, &enc)) >= 0x80) {
+	    if (mb == ENC_CODERANGE_MULTI) {
+		tokaddmbc(c, enc);
+	    }
+	    else {
+		tokadd(c);
+	    }
 	}
 	else {


-- 
Nobu Nakada