ruby-core

Nobuyoshi Nakada wrote:

> If the current encoding is not based on the Unicode character
> set, I think \u should:
> 
> a. raise compile error,
> b. be converted from Unicode to the encoding, or
> c. make the whole string UTF-8 encoding (and raise compile error
>    if other non-ascii characters are there).

The new patch attached here does c.  Any string with a \u in it is 
forced to utf-8 encoding.  Unless the \u encodes a character in the 
ASCII range, in which case it leaves the encoding alone.

>> +                       int maxbytes = rb_enc_mbmaxlen(parser->enc);
>> +                       UChar buf[maxbytes];
> 
> C99 feature can't compile with C90 compilers.

This code is gone.

\u{...} no longer accepts codepoints > 10FFFF.

This patch seems to work with regexps and %w{} as well as double-quoted 
strings and here docs.

	David

Attachments (1)

unicode_patch3 (3.79 KB, text/x-diff)

Index: parse.y
===================================================================
--- parse.y	(revision 13739)
+++ parse.y	(working copy)
@@ -237,6 +237,8 @@
     int has_shebang;
     int parser_ruby_sourceline;	/* current line no. */
     rb_encoding *enc;
+    rb_encoding *utf8;
+    int has_utf8; /* if a string contains \u escape, force encoding to utf-8*/
 
 #ifndef RIPPER
     /* Ruby core only */
@@ -264,7 +266,7 @@
 #define STR_NEW0() rb_enc_str_new(0,0,rb_enc_from_index(0))
 #define STR_NEW2(p) rb_enc_str_new((p),strlen(p),parser->enc)
 #define STR_NEW3(p,n,m) parser_str_new((p),(n),STR_ENC(!ENC_SINGLE(m)),(m))
-#define STR_ENC(m) ((m)?parser->enc:rb_enc_from_index(0))
+#define STR_ENC(m) (parser->has_utf8?parser->utf8:((m)?parser->enc:rb_enc_from_index(0)))
 #define ENC_SINGLE(cr) ((cr)==ENC_CODERANGE_SINGLE)
 #define TOK_INTERN(mb) rb_intern3(tok(), toklen(), STR_ENC(mb))
 
@@ -4675,6 +4677,7 @@
     }
 
     parser->enc = rb_enc_get(lex_input);
+    parser->utf8 = rb_enc_find("utf-8");
     ruby_sourcefile = rb_source_filename(f);
     ruby_sourceline = line - 1;
     parser_prepare(parser);
@@ -5152,7 +5155,7 @@
 #define STR_FUNC_INDENT 0x20
 
 enum string_type {
-    str_squote = (0),
+    str_squote= (0),
     str_dquote = (STR_FUNC_EXPAND),
     str_xquote = (STR_FUNC_EXPAND),
     str_regexp = (STR_FUNC_REGEXP|STR_FUNC_ESCAPE|STR_FUNC_EXPAND),
@@ -5180,6 +5183,7 @@
     } while (--len > 0 && (c = nextc()) != -1);
 }
 
+
 #define tokadd_mbchar(c) parser_tokadd_mbchar(parser, c)
 
 static int
@@ -5219,6 +5223,72 @@
 		if (func & STR_FUNC_ESCAPE) tokadd(c);
 		break;
 
+	      case 'u':
+		if ((func & STR_FUNC_EXPAND) == 0) {
+		    tokadd('\\');
+		    break;
+		}
+		else {
+		    int numlen, brace, codepoint;
+		    brace = nextc();
+		    if (brace == '{') {  /* handle \u{...} form */
+			codepoint = scan_hex(lex_p, 6, &numlen);
+			if (numlen == 0)  {
+			    yyerror("Invalid Unicode escape");
+			    return 0;
+			}
+			if (codepoint > 0x10ffff) {
+			    yyerror("Illegal Unicode codepoint (too large)");
+			    return 0;
+			}
+			lex_p += numlen;
+			
+			if ((brace = nextc()) != '}') {
+			    pushback(brace);
+			    yyerror("Unterminated Unicode escape");
+			    return 0;
+			}
+		    }
+		    else {                /* handle \uxxxx form */
+			pushback(brace);
+			codepoint = scan_hex(lex_p, 4, &numlen);
+			if (numlen < 4) {
+			    yyerror("Invalid Unicode escape");
+			    return 0;
+			}
+			lex_p += 4;
+		    }
+		
+		    if (codepoint >= 0x80) {
+			if (mb) *mb = ENC_CODERANGE_UNKNOWN;
+			parser->has_utf8 = 1;
+		    }
+		    
+		    if (codepoint < 0x80) { /* this case shouldn't happen */
+			tokadd(codepoint);
+		    }
+		    else if (codepoint < 0x800) {
+			tokadd(((codepoint >> 6)&0x1f) | 0xC0);
+			tokadd((codepoint & 0x3F) | 0x80);
+		    }
+		    else if (codepoint < 0x10000) {
+			tokadd(((codepoint >> 12) & 0x0f) | 0xe0);
+			tokadd(((codepoint >> 6)&0x3f) | 0x80);
+			tokadd((codepoint & 0x3F) | 0x80);
+		    }
+		    else if (codepoint < 0x110000) {
+			tokadd(((codepoint >> 18) & 0x07) | 0xf0);
+			tokadd(((codepoint >> 12) & 0x3f) | 0x80);
+			tokadd(((codepoint >> 6)&0x3f) | 0x80);
+			tokadd((codepoint & 0x3F) | 0x80);
+		    }
+		    else {
+			/* should not happen */
+			yyerror("Invalid Unicode codepoint");  
+		    }
+		    continue;
+		}
+
 	      default:
 		if (func & STR_FUNC_REGEXP) {
 		    pushback(c);
@@ -5301,6 +5371,7 @@
 	tokadd('#');
     }
     pushback(c);
+    parser->has_utf8 = 0;
     if (tokadd_string(func, term, paren, &quote->nd_nest, &mb) == -1) {
 	if (func & STR_FUNC_REGEXP) {
 	    ruby_sourceline = nd_line(quote);
@@ -5493,6 +5564,7 @@
 	}
 	do {
 	    pushback(c);
+	    parser->has_utf8 = 0;
 	    if ((c = tokadd_string(func, '\n', 0, NULL, mbp)) == -1) goto error;
 	    if (c != '\n') {
 		set_yylval_str(STR_NEW3(tok(), toklen(), mb));

Thread

Prev Next

In This Thread

Prev Next

[#12312] Need Japanese Help - VRuby & new One-Click Ruby Installer with patch 110 — "Curt Hibbs" <curt.hibbs@...>

[#12328] Dir.chdir patch for MS Windows — "Berger, Daniel" <Daniel.Berger@...>

[#12336] Re: gc.c -- possible logic error? — Brent Roman <brent@...>

[#12338] gc.c refactoring — Hugh Sasse <hgs@...>

[#12344] patch to implement Array.permutation — David Flanagan <david@...>

[#12346] a patch for Array.permutation and a generalization of Array.product — David Flanagan <david@...>

[#12351] ruby19 omits program name for missing files — Eric Hodel <drbrain@...7.net>

[#12352] system and Errno::ENOENT in 1.9 — Eric Hodel <drbrain@...7.net>

[#12371] Bug 6400: Time#+ and Time#- not honoring subclass — "Ron Leisti" <ron.leisti@...>

[#12372] Release compatibility/train — Prashant Srinivasan <Prashant.Srinivasan@...>

[#12375] some typos in doc/comments in repos/ruby/trunk — Eugene Ossintsev <eugoss@...>

[#12379] discrepancy between String#oct and String#hex — Ryan Davis <ryand-ruby@...>

[#12383] Include Rake in Ruby 1.9 — "NAKAMURA, Hiroshi" <nakahiro@...>

[#12401] tIDENTIFIER and is_local_id() — Thomas Enebo <Thomas.Enebo@...>

[#12404] ruby-1.9 build Sun-sparc-solaris2.9 — Hugh Sasse <hgs@...>

[#12411] rdoc patch — David Flanagan <david@...>

[#12412] REXML + RSS — Sean Russell <ser@...>

[#12414] Performance Issues with nil, true, false as Hash Keys (1.8.6) — =?ISO-8859-15?Q?Matthias_W=E4chter?= <matthias@...>

[#12418] Patch for configure.in — "Richard Brown" <rbrown@...>

[#12529] Re: Compile error for HEAD — "NARUSE, Yui" <naruse@...>

[#12533] Quick Enumerable#sort_by internals question — Daniel Berger <Daniel.Berger@...>

[#12537] Re: Why does def return nil? — "Jonas Pfenniger" <zimbatm@...>

[#12539] Ordered Hashes in 1.9? — Michael Neumann <mneumann@...>

[#12540] Re: Why does def return nil? — "Jonas Pfenniger" <zimbatm@...>

[#12547] Bug in win32ole — "Waters, Chris" <cwaters@...>

[#12550] to let Ruby 1.9 to process signals while blocked in ext/readline — Brent Roman <brent@...>

[#12564] Re: net/http.rb, patch, possible bug, test problem — Hugh Sasse <hgs@...>

[#12566] Revised net/http patch — Hugh Sasse <hgs@...>

[#12568] $" and require — "Tim Morgan" <tmorgan99@...>

[#12573] File::Stat#grpowned? is wrong — Bertram Scharpf <lists@...>

[#12575] Marshal.dump doesn't dump string encoding — David Flanagan <david@...>

[#12578] Possible memory leak in ruby-1.8.6-p110?? — "M. Edward (Ed) Borasky" <znmeb@...>

[#12579] iconv enhancement in Ruby 1.9 — "Eugene Ossintsev" <eugoss@...>

[#12581] ruby.exe exit code 255 — "Drew Sears" <dsears@...>

[#12584] Broken Ruby/ODBC due rbconfig COMMON_MACROS — "Luis Lavena" <luislavena@...>

[#12587] Confusion about arities — Charles Oliver Nutter <charles.nutter@...>

[#12588] MatchData#select rdoc and arity incorrect — Charles Oliver Nutter <charles.nutter@...>

[#12598] adds __file__ and __line__ methods to Methods and Procs — Brent Roman <brent@...>

[#12599] private method dispatch changes dropped from 1.9? — David Flanagan <david@...>

[#12610] Re: Release compatibility/train — Yukihiro Matsumoto <matz@...>

[#12615] Re: Why does def return nil? — Brent Roman <brent@...>

[#12617] Question about heap_slots in gc.c — Hongli Lai <h.lai@...>

[#12618] StringIO is not IO? — Hongli Lai <h.lai@...>

[#12623] Behaviour of String Objects - Why Not "Each" for Characters and Enumerable? — =?ISO-8859-15?Q?Wolfgang_N=E1dasi-Donner?= <ed.odanow@...>

[#12629] file encoding comments and a patch to parse.y — David Flanagan <david@...>

[#12632] Defining unicode methods — "Daniel Berger" <djberg96@...>

[#12636] Proc.yield method — David Flanagan <david@...>

[#12638] New lambda -> syntax — David Flanagan <david@...>

[#12639] test problem after compiling — Martin Duerst <duerst@...>

[#12664] minor rdoc patch to encoding.c — David Flanagan <david@...>

[#12665] Another encoding.c patch — David Flanagan <david@...>

[#12666] encoding.c bug — David Flanagan <david@...>

[#12670] Bug in Numeric#divmod — "Dirk Traulsen" <dirk.traulsen@...>

[#12675] Re: Import gem to Ruby 1.9 — Brent Roman <brent@...>

[#12676] Minor Array.shuffle patch — David Flanagan <david@...>

[#12679] documentation patch for enum.c — David Flanagan <david@...>

[#12681] Unicode: Progress? — murphy <murphy@...>

[#12693] retry: revised 1.9 http patch — Hugh Sasse <hgs@...>

[#12695] New Rails plugin 'Liftoff' is available. — "Ben Tompkins" <nbitspoken@...>

[#12697] Range.first is incompatible with Enumerable.first — David Flanagan <david@...>

[#12703] Long encoding names with -K and bad error message — David Flanagan <david@...>

[#12705] A couple of bugs? — "John Lam (DLR)" <jflam@...>

[#12706] Re: A couple of bugs? — "Gavin Kistner" <gavin.kistner@...>

[#12710] enum.c patch: fixes Enumerable.cycle and rdoc bugs — David Flanagan <david@...>

[#12714] Re: A couple of bugs? — "Gavin Kistner" <gavin.kistner@...>

[#12750] Re: [OT] Re: Should *most* memory be release back to the system? — "M. Edward (Ed) Borasky" <znmeb@...>

[#12754] Improving 'syntax error, unexpected $end, expecting kEND'? — Hugh Sasse <hgs@...>

[#12757] Re: Unicode: Progress?? — Brent Roman <brent@...>

[#12758] Encoding::primary_encoding — David Flanagan <david@...>

[#12759] Reading UTF-8 Data contains BOM — =?ISO-8859-15?Q?Wolfgang_N=E1dasi-Donner?= <ed.odanow@...>

[#12764] SystemStackError 1.8.6 (stack only 25 frames) — Dan Janowski <danj@3skel.com>

[#12767] \u escapes in string literals: proof of concept implementation — David Flanagan <david@...>

[#12776] Clarification of Ruby release plans? 1.8.7? 1.9.1? 2.0? — Lucas Nussbaum <lucas@...>

[#12779] Re: Unicode: Progress?? — Brent Roman <brent@...>

[#12780] Re: Unicode: Progress?? — Brent Roman <brent@...>

[#12787] How to specify in Ruby 1.9 the expected file encoding — =?ISO-8859-15?Q?Wolfgang_N=E1dasi-Donner?= <ed.odanow@...>

[#12795] patch for String.concat — David Flanagan <david@...>

[#12825] clarification of ruby libraries installation paths? — Lucas Nussbaum <lucas@...>

[#12834] Bug Tracker Access Request — James Edward Gray II <james@...>

[#12847] make test-all — "Florian Ebeling" <florian.ebeling@...>

[#12598] adds file and line methods to Methods and Procs — Brent Roman <brent@...>

[#12750] Re: [OT] Re: Should most memory be release back to the system? — "M. Edward (Ed) Borasky" <znmeb@...>