Re: \u escapes in string literals: proof of concept implementation

Hi,

At Tue, 23 Oct 2007 13:10:40 +0900,
David Flanagan wrote in [ruby-core:12864]:
> > That things are simpler is quite clear. But option a wouldn't be
> > difficult to implement, either, I guess. My suggestion is to stay
> > with option a until we have a better idea of which of b and c is
> > really needed/implementable/... 
> 
> The appeal, to me, of my current version is that it is independent of 
> the primary encoding.  \u escapes work no matter what -K option you 
> specify, and they always translate to a specific byte sequence. On the 
> other hand, I really don't know how to handle strings that mix 
> sjis-encoded Kanji characters with \u escapes.  What should the encoding 
> of the resulting string be?

That's garbage.

> So chosing option a might be the best bet: \u escapes cause an error 
> with -Ks or -Ke.  They are only allowed when the primary encoding is 
> ascii or utf-8.  I think that would mean that no transcoding would be 
> necessary.  I also think that my current patch doesn't need much 
> modification: just the addition of errors when the encoding does not 
> allow \u.

Another option is c of [ruby-core:12769].


Index: parse.y
===================================================================
--- parse.y	(revision 13774)
+++ parse.y	(working copy)
@@ -238,4 +238,5 @@ struct parser_params {
     int parser_ruby_sourceline;	/* current line no. */
     rb_encoding *enc;
+    rb_encoding *utf8;
 
 #ifndef RIPPER
@@ -261,8 +262,11 @@ struct parser_params {
 };
 
+#define UTF8_ENC() (parser->utf8 ? parser->utf8 : \
+		    (parser->utf8 = rb_enc_find("utf-8")))
 #define STR_NEW(p,n) rb_enc_str_new((p),(n),parser->enc)
 #define STR_NEW0() rb_str_new(0,0)
 #define STR_NEW2(p) rb_enc_str_new((p),strlen(p),parser->enc)
 #define STR_NEW3(p,n,m) parser_str_new((p),(n),STR_ENC(!ENC_SINGLE(m)),(m))
+#define STR_NEW4(p,n,e,m) parser_str_new((p),(n), (e), (m))
 #define STR_ENC(m) ((m)?parser->enc:rb_enc_from_index(0))
 #define ENC_SINGLE(cr) ((cr)==ENC_CODERANGE_SINGLE)
@@ -4488,5 +4492,5 @@ none		: /* none */
 
 static int parser_regx_options(struct parser_params*);
-static int parser_tokadd_string(struct parser_params*,int,int,int,long*,int*);
+static int parser_tokadd_string(struct parser_params*,int,int,int,long*,int*,rb_encoding**);
 static int parser_parse_string(struct parser_params*,NODE*);
 static int parser_here_document(struct parser_params*,NODE*);
@@ -4497,8 +4501,10 @@ static int parser_here_document(struct p
 # define tokspace(n)               parser_tokspace(parser, n)
 # define tokadd(c)                 parser_tokadd(parser, c)
-# define read_escape(m)            parser_read_escape(parser, m)
-# define tokadd_escape(t,m)        parser_tokadd_escape(parser, t, m)
+# define tok_hex(numlen)           parser_tok_hex(parser, numlen)
+# define tok_utf8(numlen,e)        parser_tok_utf8(parser, numlen, e)
+# define read_escape(flags,m,e)    parser_read_escape(parser, flags, m, e)
+# define tokadd_escape(t,m,e)      parser_tokadd_escape(parser, t, m, e)
 # define regx_options()            parser_regx_options(parser)
-# define tokadd_string(f,t,p,n,m)  parser_tokadd_string(parser,f,t,p,n,m)
+# define tokadd_string(f,t,p,n,m,e) parser_tokadd_string(parser,f,t,p,n,m, e)
 # define parse_string(n)           parser_parse_string(parser,n)
 # define here_document(n)          parser_here_document(parser,n)
@@ -4938,5 +4944,73 @@ parser_tokadd(struct parser_params *pars
 
 static int
-parser_read_escape(struct parser_params *parser, int *mb)
+parser_tok_hex(struct parser_params *parser, int *numlen)
+{
+    int c;
+
+    if (peek('{')) {
+	nextc();
+	c = scan_hex(lex_p, 8, numlen);
+	if (!*numlen) goto invalid;
+	if (!peek('}')) {
+	    yyerror("unterminated hex escape");
+	    return 0;
+	}
+	nextc();
+	*numlen += 2;
+    }
+    else {
+	c = scan_hex(lex_p, 2, numlen);
+	if (!*numlen) {
+	  invalid:
+	    yyerror("invalid hex escape");
+	    return 0;
+	}
+    }
+    return c;
+}
+
+static int
+parser_tok_utf8(struct parser_params *parser, int *numlen, rb_encoding **encp)
+{
+    int codepoint;
+
+    if (peek('{')) {  /* handle \u{...} form */
+	nextc();
+	codepoint = scan_hex(lex_p, 8, numlen);
+	if (*numlen == 0)  {
+	    yyerror("invalid Unicode escape");
+	    return 0;
+	}
+	if (codepoint > 0x7fffffff) {
+	    yyerror("illegal Unicode codepoint (too large)");
+	    return 0;
+	}
+	lex_p += *numlen;
+	if (!peek('}')) {
+	    yyerror("unterminated Unicode escape");
+	    return 0;
+	}
+	nextc();
+    }
+    else {			/* handle \uxxxx form */
+	codepoint = scan_hex(lex_p, 4, numlen);
+	if (*numlen < 4) {
+	    yyerror("invalid Unicode escape");
+	    return 0;
+	}
+	lex_p += 4;
+    }
+    if (codepoint >= 0x80) {
+	*encp = UTF8_ENC();
+    }
+
+    return codepoint;
+}
+
+#define ESCAPE_CONTROL 1
+#define ESCAPE_META    2
+
+static int
+parser_read_escape(struct parser_params *parser, int flags, int *mb, rb_encoding **encp)
 {
     int c;
@@ -4969,4 +5043,5 @@ parser_read_escape(struct parser_params 
       case '0': case '1': case '2': case '3': /* octal constant */
       case '4': case '5': case '6': case '7':
+	if (flags & (ESCAPE_CONTROL|ESCAPE_META)) goto eof;
 	{
 	    int numlen;
@@ -4980,13 +5055,21 @@ parser_read_escape(struct parser_params 
 
       case 'x':	/* hex constant */
+	if (flags & (ESCAPE_CONTROL|ESCAPE_META)) goto eof;
 	{
 	    int numlen;
 
-	    c = scan_hex(lex_p, 2, &numlen);
-	    if (numlen == 0) {
-		yyerror("Invalid escape character syntax");
-		return 0;
-	    }
-	    lex_p += numlen;
+	    c = tok_hex(&numlen);
+	    if (numlen == 0) goto eof;
+	}
+	if (mb && (c >= 0x80)) *mb = ENC_CODERANGE_UNKNOWN;
+	return c;
+
+      case 'u':	/* hex constant */
+	if (flags & (ESCAPE_CONTROL|ESCAPE_META)) goto eof;
+	{
+	    int numlen;
+
+	    c = tok_utf8(&numlen, encp);
+	    if (numlen == 0) goto eof;
 	}
 	if (mb && (c >= 0x80)) *mb = ENC_CODERANGE_UNKNOWN;
@@ -5000,12 +5083,12 @@ parser_read_escape(struct parser_params 
 
       case 'M':
+	if (flags & ESCAPE_META) goto eof;
 	if ((c = nextc()) != '-') {
-	    yyerror("Invalid escape character syntax");
 	    pushback(c);
-	    return '\0';
+	    goto eof;
 	}
 	if ((c = nextc()) == '\\') {
 	    if (mb) *mb = ENC_CODERANGE_UNKNOWN;
-	    return read_escape(0) | 0x80;
+	    return read_escape(flags|ESCAPE_META, 0, encp) | 0x80;
 	}
 	else if (c == -1) goto eof;
@@ -5017,11 +5100,11 @@ parser_read_escape(struct parser_params 
       case 'C':
 	if ((c = nextc()) != '-') {
-	    yyerror("Invalid escape character syntax");
 	    pushback(c);
-	    return '\0';
+	    goto eof;
 	}
       case 'c':
+	if (flags & ESCAPE_CONTROL) goto eof;
 	if ((c = nextc())== '\\') {
-	    c = read_escape(mb);
+	    c = read_escape(flags|ESCAPE_CONTROL, mb, encp);
 	}
 	else if (c == '?')
@@ -5040,9 +5123,13 @@ parser_read_escape(struct parser_params 
 }
 
+#define tokcopy(n) memcpy(tokspace(n), lex_p - (n), (n))
+
 static int
-parser_tokadd_escape(struct parser_params *parser, int term, int *mb)
+parser_tokadd_escape(struct parser_params *parser, int term, int *mb, rb_encoding **encp)
 {
     int c;
+    int flags = 0;
 
+  first:
     switch (c = nextc()) {
       case '\n':
@@ -5051,17 +5138,13 @@ parser_tokadd_escape(struct parser_param
       case '0': case '1': case '2': case '3': /* octal constant */
       case '4': case '5': case '6': case '7':
+	if (flags & (ESCAPE_CONTROL|ESCAPE_META)) goto eof;
 	{
 	    int numlen;
 	    int oct;
 
-	    tokadd('\\');
-	    pushback(c);
-	    oct = scan_oct(lex_p, 3, &numlen);
-	    if (numlen == 0) {
-		yyerror("Invalid escape character syntax");
-		return -1;
-	    }
-	    while (numlen--)
-		tokadd(nextc());
+	    oct = scan_oct(--lex_p, 3, &numlen);
+	    if (numlen == 0) goto eof;
+	    lex_p += numlen;
+	    tokcopy(numlen + 1);
 	    if (mb && (oct >= 0200)) *mb = ENC_CODERANGE_UNKNOWN;
 	}
@@ -5069,45 +5152,59 @@ parser_tokadd_escape(struct parser_param
 
       case 'x':	/* hex constant */
+	if (flags & (ESCAPE_CONTROL|ESCAPE_META)) goto eof;
 	{
 	    int numlen;
 	    int hex;
 
-	    tokadd('\\');
-	    tokadd(c);
-	    hex = scan_hex(lex_p, 2, &numlen);
-	    if (numlen == 0) {
-		yyerror("Invalid escape character syntax");
-		return -1;
-	    }
-	    while (numlen--)
-		tokadd(nextc());
+	    hex = tok_hex(&numlen);
+	    if (numlen == 0) goto eof;
+	    lex_p += numlen;
+	    tokcopy(numlen + 2);
 	    if (mb && (hex >= 0x80)) *mb = ENC_CODERANGE_UNKNOWN;
 	}
 	return 0;
 
+      case 'u':	/* Unicode constant */
+	if (flags & (ESCAPE_CONTROL|ESCAPE_META)) goto eof;
+	{
+	    int numlen;
+	    int uc;
+
+	    uc = tok_utf8(&numlen, encp);
+	    if (numlen == 0) goto eof;
+	    lex_p += numlen;
+	    tokcopy(numlen + 2);
+	    if (mb && (uc >= 0x80)) *mb = ENC_CODERANGE_MULTI;
+	    if (uc >= 0x80) return 1;
+	}
+	return 0;
+
       case 'M':
+	if (flags & ESCAPE_META) goto eof;
 	if ((c = nextc()) != '-') {
-	    yyerror("Invalid escape character syntax");
 	    pushback(c);
-	    return 0;
+	    goto eof;
 	}
-	tokadd('\\'); tokadd('M'); tokadd('-');
+	tokcopy(3);
 	if (mb) *mb = ENC_CODERANGE_UNKNOWN;
+	flags |= ESCAPE_META;
 	goto escaped;
 
       case 'C':
+	if (flags & ESCAPE_CONTROL) goto eof;
 	if ((c = nextc()) != '-') {
-	    yyerror("Invalid escape character syntax");
 	    pushback(c);
-	    return 0;
+	    goto eof;
 	}
-	tokadd('\\'); tokadd('C'); tokadd('-');
+	tokcopy(3);
 	goto escaped;
 
       case 'c':
-	tokadd('\\'); tokadd('c');
+	if (flags & ESCAPE_CONTROL) goto eof;
+	tokcopy(2);
+	flags |= ESCAPE_CONTROL;
       escaped:
 	if ((c = nextc()) == '\\') {
-	    return tokadd_escape(term, mb);
+	    goto first;
 	}
 	else if (c == -1) goto eof;
@@ -5190,16 +5287,47 @@ parser_tokadd_mbchar(struct parser_param
 {
     int len = parser_mbclen();
-    do {
-	tokadd(c);
-    } while (--len > 0 && (c = nextc()) != -1);
+    tokadd(c);
+    lex_p += --len;
+    if (len > 0) tokcopy(len);
 }
 
 #define tokadd_mbchar(c) parser_tokadd_mbchar(parser, c)
 
+static void
+parser_tokaddmbc(struct parser_params *parser, int c, rb_encoding *enc)
+{
+    int len = rb_enc_codelen(c, enc);
+    rb_enc_mbcput(c, tokspace(len), enc);
+}
+
+#define tokaddmbc(c, enc) parser_tokaddmbc(parser, c, enc)
+
 static int
 parser_tokadd_string(struct parser_params *parser,
-		     int func, int term, int paren, long *nest, int *mb)
+		     int func, int term, int paren, long *nest,
+		     int *mb, rb_encoding **encp)
 {
     int c;
+    int has_mb = 0;
+    rb_encoding *enc = *encp;
+    char *errbuf = 0;
+    static const char mixed_msg[] = "%s mixed within %s source";
+
+#define mixed_error(enc1, enc2) if (!errbuf) {	\
+	int len = sizeof(mixed_msg) - 4;	\
+	len += strlen(rb_enc_name(enc1));	\
+	len += strlen(rb_enc_name(enc2));	\
+	errbuf = ALLOCA_N(char, len);		\
+	snprintf(errbuf, len, mixed_msg,	\
+		 rb_enc_name(enc1),		\
+		 rb_enc_name(enc2));		\
+	yyerror(errbuf);			\
+    }
+#define mixed_escape(beg, enc1, enc2) do {	\
+	const char *pos = lex_p;		\
+	lex_p = beg;				\
+	mixed_error(enc1, enc2);		\
+	lex_p = pos;				\
+    } while (0)
 
     while ((c = nextc()) != -1) {
@@ -5222,4 +5350,5 @@ parser_tokadd_string(struct parser_param
 	}
 	else if (c == '\\') {
+	    const char *beg = lex_p - 1;
 	    c = nextc();
 	    switch (c) {
@@ -5237,6 +5366,9 @@ parser_tokadd_string(struct parser_param
 		if (func & STR_FUNC_REGEXP) {
 		    pushback(c);
-		    if (tokadd_escape(term, mb) < 0)
+		    if ((c = tokadd_escape(term, mb, &enc)) < 0)
 			return -1;
+		    if (has_mb && enc != *encp) {
+			mixed_escape(beg, enc, *encp);
+		    }
 		    continue;
 		}
@@ -5244,5 +5376,13 @@ parser_tokadd_string(struct parser_param
 		    pushback(c);
 		    if (func & STR_FUNC_ESCAPE) tokadd('\\');
-		    c = read_escape(mb);
+		    c = read_escape(0, mb, &enc);
+		    if (has_mb && enc != *encp) {
+			mixed_escape(beg, enc, *encp);
+			continue;
+		    }
+		    if (c >= 0x80) {
+			tokaddmbc(c, enc);
+			continue;
+		    }
 		}
 		else if ((func & STR_FUNC_QWORDS) && ISSPACE(c)) {
@@ -5255,4 +5395,9 @@ parser_tokadd_string(struct parser_param
 	}
 	else if (parser_ismbchar()) {
+	    has_mb = 1;
+	    if (enc != *encp) {
+		mixed_error(enc, *encp);
+		continue;
+	    }
 	    tokadd_mbchar(c);
 	    if (mb) *mb = ENC_CODERANGE_MULTI;
@@ -5270,4 +5415,5 @@ parser_tokadd_string(struct parser_param
 	tokadd(c);
     }
+    *encp = enc;
     return c;
 }
@@ -5283,4 +5429,5 @@ parser_parse_string(struct parser_params
     int paren = nd_paren(quote);
     int c, space = 0, mb = ENC_CODERANGE_SINGLE;
+    rb_encoding *enc = parser->enc;
 
     if (func == -1) return tSTRING_END;
@@ -5316,12 +5463,11 @@ parser_parse_string(struct parser_params
     }
     pushback(c);
-    if (tokadd_string(func, term, paren, &quote->nd_nest, &mb) == -1) {
+    if (tokadd_string(func, term, paren, &quote->nd_nest, &mb, &enc) == -1) {
+	ruby_sourceline = nd_line(quote);
 	if (func & STR_FUNC_REGEXP) {
-	    ruby_sourceline = nd_line(quote);
 	    compile_error(PARSER_ARG "unterminated regexp meets end of file");
 	    return tREGEXP_END;
 	}
 	else {
-	    ruby_sourceline = nd_line(quote);
 	    compile_error(PARSER_ARG "unterminated string meets end of file");
 	    return tSTRING_END;
@@ -5330,5 +5476,5 @@ parser_parse_string(struct parser_params
 
     tokfix();
-    set_yylval_str(STR_NEW3(tok(), toklen(), mb));
+    set_yylval_str(STR_NEW4(tok(), toklen(), enc, mb));
     return tSTRING_CONTENT;
 }
@@ -5494,4 +5640,5 @@ parser_here_document(struct parser_param
     else {
 	int mb = ENC_CODERANGE_SINGLE, *mbp = &mb;
+	rb_encoding *enc = parser->enc;
 	newtok();
 	if (c == '#') {
@@ -5508,7 +5655,7 @@ parser_here_document(struct parser_param
 	do {
 	    pushback(c);
-	    if ((c = tokadd_string(func, '\n', 0, NULL, mbp)) == -1) goto error;
+	    if ((c = tokadd_string(func, '\n', 0, NULL, mbp, &enc)) == -1) goto error;
 	    if (c != '\n') {
-		set_yylval_str(STR_NEW3(tok(), toklen(), mb));
+		set_yylval_str(STR_NEW4(tok(), toklen(), enc, mb));
 		return tSTRING_CONTENT;
 	    }
@@ -5517,5 +5664,5 @@ parser_here_document(struct parser_param
 	    if ((c = nextc()) == -1) goto error;
 	} while (!whole_match_p(eos, len, indent));
-	str = STR_NEW3(tok(), toklen(), mb);
+	str = STR_NEW4(tok(), toklen(), enc, mb);
     }
     heredoc_restore(lex_strterm);
@@ -5778,4 +5925,5 @@ parser_yylex(struct parser_params *parse
     enum lex_state_e last_state;
     int mb;
+    rb_encoding *enc;
 #ifdef RIPPER
     int fallthru = Qfalse;
@@ -6099,4 +6247,5 @@ parser_yylex(struct parser_params *parse
 	}
 	newtok();
+	enc = parser->enc;
 	if (parser_ismbchar()) {
 	    mb = ENC_CODERANGE_MULTI;
@@ -6107,8 +6256,7 @@ parser_yylex(struct parser_params *parse
 	    goto ternary;
 	}
-	else if (c == '\\' && (c = read_escape(0)) >= 0x80) {
-	    rb_encoding *enc = parser->enc;
+	else if (c == '\\' && (c = read_escape(0, 0, &enc)) >= 0x80) {
 	    mb = ENC_CODERANGE_UNKNOWN;
-	    rb_enc_mbcput(c, tokspace(rb_enc_codelen(c, enc)), enc);
+	    tokaddmbc(c, enc);
 	}
 	else {
@@ -6117,5 +6265,5 @@ parser_yylex(struct parser_params *parse
 	}
 	tokfix();
-	set_yylval_str(STR_NEW3(tok(), toklen(), mb));
+	set_yylval_str(parser_str_new(tok(), toklen(), enc, mb));
 	lex_state = EXPR_ENDARG;
 	return tCHAR;
@@ -7187,4 +7335,15 @@ list_concat_gen(struct parser_params *pa
 }
 
+static void
+literal_concat0(struct parser_params *parser, VALUE head, VALUE tail)
+{
+    if (!rb_enc_compatible(head, tail)) {
+	compile_error(PARSER_ARG "string literal encodings differ (%s / %s)",
+		      rb_enc_name(rb_enc_get(head)),
+		      rb_enc_name(rb_enc_get(tail)));
+    }
+    rb_str_buf_append(head, tail);
+}
+
 /* concat two string literals */
 static NODE *
@@ -7204,5 +7363,5 @@ literal_concat_gen(struct parser_params 
       case NODE_STR:
 	if (htype == NODE_STR) {
-	    rb_str_concat(head->nd_lit, tail->nd_lit);
+	    literal_concat0(parser, head->nd_lit, tail->nd_lit);
 	    rb_gc_force_recycle((VALUE)tail);
 	}
@@ -7214,5 +7373,5 @@ literal_concat_gen(struct parser_params 
       case NODE_DSTR:
 	if (htype == NODE_STR) {
-	    rb_str_concat(head->nd_lit, tail->nd_lit);
+	    literal_concat0(parser, head->nd_lit, tail->nd_lit);
 	    tail->nd_lit = head->nd_lit;
 	    rb_gc_force_recycle((VALUE)head);


-- 
Nobu Nakada