From: David Flanagan <david@...>
Date: 2007-10-20T09:50:58+09:00
Subject: Re: \u escapes in string literals: proof of concept implementation

This is a multi-part message in MIME format.
--------------000201050103030704030703
Content-Type: text/plain; charset=ISO-8859-1; format=flowed
Content-Transfer-Encoding: 7bit

Nobuyoshi Nakada wrote:

> If the current encoding is not based on the Unicode character
> set, I think \u should:
> 
> a. raise compile error,
> b. be converted from Unicode to the encoding, or
> c. make the whole string UTF-8 encoding (and raise compile error
>    if other non-ascii characters are there).

The new patch attached here does c.  Any string with a \u in it is 
forced to utf-8 encoding.  Unless the \u encodes a character in the 
ASCII range, in which case it leaves the encoding alone.

>> +                       int maxbytes = rb_enc_mbmaxlen(parser->enc);
>> +                       UChar buf[maxbytes];
> 
> C99 feature can't compile with C90 compilers.

This code is gone.

\u{...} no longer accepts codepoints > 10FFFF.

This patch seems to work with regexps and %w{} as well as double-quoted 
strings and here docs.

	David


--------------000201050103030704030703
Content-Type: text/plain;
 name="unicode_patch3"
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="unicode_patch3"

Index: parse.y
===================================================================
--- parse.y	(revision 13739)
+++ parse.y	(working copy)
@@ -237,6 +237,8 @@
     int has_shebang;
     int parser_ruby_sourceline;	/* current line no. */
     rb_encoding *enc;
+    rb_encoding *utf8;
+    int has_utf8; /* if a string contains \u escape, force encoding to utf-8*/
 
 #ifndef RIPPER
     /* Ruby core only */
@@ -264,7 +266,7 @@
 #define STR_NEW0() rb_enc_str_new(0,0,rb_enc_from_index(0))
 #define STR_NEW2(p) rb_enc_str_new((p),strlen(p),parser->enc)
 #define STR_NEW3(p,n,m) parser_str_new((p),(n),STR_ENC(!ENC_SINGLE(m)),(m))
-#define STR_ENC(m) ((m)?parser->enc:rb_enc_from_index(0))
+#define STR_ENC(m) (parser->has_utf8?parser->utf8:((m)?parser->enc:rb_enc_from_index(0)))
 #define ENC_SINGLE(cr) ((cr)==ENC_CODERANGE_SINGLE)
 #define TOK_INTERN(mb) rb_intern3(tok(), toklen(), STR_ENC(mb))
 
@@ -4675,6 +4677,7 @@
     }
 
     parser->enc = rb_enc_get(lex_input);
+    parser->utf8 = rb_enc_find("utf-8");
     ruby_sourcefile = rb_source_filename(f);
     ruby_sourceline = line - 1;
     parser_prepare(parser);
@@ -5152,7 +5155,7 @@
 #define STR_FUNC_INDENT 0x20
 
 enum string_type {
-    str_squote = (0),
+    str_squote= (0),
     str_dquote = (STR_FUNC_EXPAND),
     str_xquote = (STR_FUNC_EXPAND),
     str_regexp = (STR_FUNC_REGEXP|STR_FUNC_ESCAPE|STR_FUNC_EXPAND),
@@ -5180,6 +5183,7 @@
     } while (--len > 0 && (c = nextc()) != -1);
 }
 
+
 #define tokadd_mbchar(c) parser_tokadd_mbchar(parser, c)
 
 static int
@@ -5219,6 +5223,72 @@
 		if (func & STR_FUNC_ESCAPE) tokadd(c);
 		break;
 
+	      case 'u':
+		if ((func & STR_FUNC_EXPAND) == 0) {
+		    tokadd('\\');
+		    break;
+		}
+		else {
+		    int numlen, brace, codepoint;
+		    brace = nextc();
+		    if (brace == '{') {  /* handle \u{...} form */
+			codepoint = scan_hex(lex_p, 6, &numlen);
+			if (numlen == 0)  {
+			    yyerror("Invalid Unicode escape");
+			    return 0;
+			}
+			if (codepoint > 0x10ffff) {
+			    yyerror("Illegal Unicode codepoint (too large)");
+			    return 0;
+			}
+			lex_p += numlen;
+			
+			if ((brace = nextc()) != '}') {
+			    pushback(brace);
+			    yyerror("Unterminated Unicode escape");
+			    return 0;
+			}
+		    }
+		    else {                /* handle \uxxxx form */
+			pushback(brace);
+			codepoint = scan_hex(lex_p, 4, &numlen);
+			if (numlen < 4) {
+			    yyerror("Invalid Unicode escape");
+			    return 0;
+			}
+			lex_p += 4;
+		    }
+		
+		    if (codepoint >= 0x80) {
+			if (mb) *mb = ENC_CODERANGE_UNKNOWN;
+			parser->has_utf8 = 1;
+		    }
+		    
+		    if (codepoint < 0x80) { /* this case shouldn't happen */
+			tokadd(codepoint);
+		    }
+		    else if (codepoint < 0x800) {
+			tokadd(((codepoint >> 6)&0x1f) | 0xC0);
+			tokadd((codepoint & 0x3F) | 0x80);
+		    }
+		    else if (codepoint < 0x10000) {
+			tokadd(((codepoint >> 12) & 0x0f) | 0xe0);
+			tokadd(((codepoint >> 6)&0x3f) | 0x80);
+			tokadd((codepoint & 0x3F) | 0x80);
+		    }
+		    else if (codepoint < 0x110000) {
+			tokadd(((codepoint >> 18) & 0x07) | 0xf0);
+			tokadd(((codepoint >> 12) & 0x3f) | 0x80);
+			tokadd(((codepoint >> 6)&0x3f) | 0x80);
+			tokadd((codepoint & 0x3F) | 0x80);
+		    }
+		    else {
+			/* should not happen */
+			yyerror("Invalid Unicode codepoint");  
+		    }
+		    continue;
+		}
+
 	      default:
 		if (func & STR_FUNC_REGEXP) {
 		    pushback(c);
@@ -5301,6 +5371,7 @@
 	tokadd('#');
     }
     pushback(c);
+    parser->has_utf8 = 0;
     if (tokadd_string(func, term, paren, &quote->nd_nest, &mb) == -1) {
 	if (func & STR_FUNC_REGEXP) {
 	    ruby_sourceline = nd_line(quote);
@@ -5493,6 +5564,7 @@
 	}
 	do {
 	    pushback(c);
+	    parser->has_utf8 = 0;
 	    if ((c = tokadd_string(func, '\n', 0, NULL, mbp)) == -1) goto error;
 	    if (c != '\n') {
 		set_yylval_str(STR_NEW3(tok(), toklen(), mb));

--------------000201050103030704030703--