From: Tanaka Akira <akr@...>
Date: 2008-01-26T12:14:00+09:00
Subject: [ruby-dev:33400] /#{}/e.encoding

以下のように /#{}/e の encoding が US-ASCII になります。

  % ./ruby -ve 'p(/#{}/e.encoding)'
  ruby 1.9.0 (2008-01-26 revision 15237) [i686-linux]
  #<Encoding:US-ASCII>

正規表現のソースを構成する文字列の連結時に EUC-JP な
encoding が消えてしまうのが理由なようです。

もともとこの連結に単なる文字列連結を使うのは、エスケープされ
た中途半端な非 ASCII 文字が連結されて中途半端でないものにな
るのを検出できないという問題があります。なので、
Regexp.preprocess というメソッドを新設して、そこで連結を行い、
エンコーディングの伝播も以前同様ににされるようにしてみました。

Index: re.c
===================================================================
--- re.c	(revision 15249)
+++ re.c	(working copy)
@@ -1942,36 +1942,51 @@ rb_reg_check_preprocess(VALUE str)
     return Qnil;
 }
 
-#if 0
 static VALUE
-rb_reg_preprocess_obj(VALUE str,
-        rb_encoding **fixed_enc, onig_errmsg_buffer err)
+rb_reg_preprocess_m(int argc, VALUE *argv, VALUE klass)
 {
-    VALUE buf;
-    char *p, *end;
-    rb_encoding *enc;
+    rb_encoding *fixed_enc = 0;
+    onig_errmsg_buffer err = "";
+    int i;
+    VALUE result = 0;
 
-    StringValue(str);
-    p = RSTRING_PTR(str);
-    end = p + RSTRING_LEN(str);
-    enc = rb_enc_get(str);
+    if (argc == 0) {
+        rb_raise(rb_eArgError, "no arguments given");
+    }
 
-    buf = rb_reg_preprocess(p, end, enc, fixed_enc, err);
-    RB_GC_GUARD(str);
-    return buf;
-}
+    for (i = 0; i < argc; i++) {
+        VALUE str = argv[i];
+        VALUE buf;
+        char *p, *end;
+        rb_encoding *enc;
+
+        StringValue(str);
+        p = RSTRING_PTR(str);
+        end = p + RSTRING_LEN(str);
+        enc = rb_enc_get(str);
+
+        buf = rb_reg_preprocess(p, end, enc, &fixed_enc, err);
+        RB_GC_GUARD(str);
+
+        if (buf == Qnil)
+            rb_raise(rb_eArgError, "%s", err);
+
+        if (i == 0) {
+            /* The encoding of the first fragment is the encoding 
+             * given by the regexp option or script encoding. */
+            if (fixed_enc == 0) {
+                rb_enc_copy(buf, str);
+            }
+        }
 
-static VALUE
-rb_reg_preprocess_m(VALUE klass, VALUE obj)
-{
-    rb_encoding *fixed_enc = 0;
-    onig_errmsg_buffer err = "";
-    VALUE str = rb_reg_preprocess_obj(obj, &fixed_enc, err);
-    if (str == Qnil)
-        rb_raise(rb_eArgError, "%s", err);
-    return rb_assoc_new(str, fixed_enc ? Qtrue : Qfalse);
+        if (!result)
+            result = buf;
+        else
+            rb_str_buf_append(result, buf);
+    }
+
+    return result;
 }
-#endif
 
 static int
 rb_reg_initialize(VALUE obj, const char *s, int len, rb_encoding *enc,
@@ -3035,9 +3050,7 @@ Init_Regexp(void)
     rb_define_singleton_method(rb_cRegexp, "last_match", rb_reg_s_last_match, -1);
     rb_define_singleton_method(rb_cRegexp, "try_convert", rb_reg_s_try_convert, 1);
 
-#if 0
-    rb_define_singleton_method(rb_cRegexp, "preprocess", rb_reg_preprocess_m, 1);
-#endif
+    rb_define_singleton_method(rb_cRegexp, "preprocess", rb_reg_preprocess_m, -1);
 
     rb_define_method(rb_cRegexp, "initialize", rb_reg_initialize_m, -1);
     rb_define_method(rb_cRegexp, "initialize_copy", rb_reg_init_copy, 1);
Index: compile.c
===================================================================
--- compile.c	(revision 15249)
+++ compile.c	(working copy)
@@ -1848,7 +1848,7 @@ iseq_set_sequence_stackcaching(rb_iseq_t
 
 
 static int
-compile_dstr(rb_iseq_t *iseq, LINK_ANCHOR *ret, NODE * node)
+compile_dstr_fragments(rb_iseq_t *iseq, LINK_ANCHOR *ret, NODE * node, int *cntp)
 {
     NODE *list = node->nd_next;
     VALUE lit = node->nd_lit;
@@ -1862,12 +1862,31 @@ compile_dstr(rb_iseq_t *iseq, LINK_ANCHO
 	cnt++;
 	list = list->nd_next;
     }
+    *cntp = cnt;
+
+    return COMPILE_OK;
+}
 
+static int
+compile_dstr(rb_iseq_t *iseq, LINK_ANCHOR *ret, NODE * node)
+{
+    int cnt;
+    compile_dstr_fragments(iseq, ret, node, &cnt);
     ADD_INSN1(ret, nd_line(node), concatstrings, INT2FIX(cnt));
     return COMPILE_OK;
 }
 
 static int
+compile_dregx(rb_iseq_t *iseq, LINK_ANCHOR *ret, NODE * node)
+{
+    int cnt;
+    ADD_INSN1(ret, nd_line(node), putobject, rb_cRegexp);
+    compile_dstr_fragments(iseq, ret, node, &cnt);
+    ADD_SEND(ret, nd_line(node), ID2SYM(rb_intern("preprocess")), INT2FIX(cnt));
+    return COMPILE_OK;
+}
+
+static int
 compile_branch_condition(rb_iseq_t *iseq, LINK_ANCHOR *ret, NODE * cond,
 			 LABEL *then_label, LABEL *else_label)
 {
@@ -4077,7 +4096,7 @@ iseq_compile_each(rb_iseq_t *iseq, LINK_
 	break;
       }
       case NODE_DREGX:{
-	compile_dstr(iseq, ret, node);
+	compile_dregx(iseq, ret, node);
 	ADD_INSN1(ret, nd_line(node), toregexp, INT2FIX(node->nd_cflag));
 
 	if (poped) {
@@ -4094,7 +4113,7 @@ iseq_compile_each(rb_iseq_t *iseq, LINK_
 	ADD_INSN2(ret, nd_line(node), onceinlinecache, 0, lend);
 	ADD_INSN(ret, nd_line(node), pop);
 
-	compile_dstr(iseq, ret, node);
+	compile_dregx(iseq, ret, node);
 	ADD_INSN1(ret, nd_line(node), toregexp, INT2FIX(node->nd_cflag));
 
 	ADD_INSN1(ret, nd_line(node), setinlinecache, lstart);
Index: test/ruby/test_m17n.rb
===================================================================
--- test/ruby/test_m17n.rb	(revision 15249)
+++ test/ruby/test_m17n.rb	(working copy)
@@ -428,14 +428,14 @@ class TestM17N < Test::Unit::TestCase
     assert_raise(ArgumentError) { eval(s("/\#{r}\xc2\xa1/s")) }
 
     r = /\xc2\xa1/e
-    #assert_raise(ArgumentError) { eval(s("/\xc2\xa1\#{r}/s")) }
-    #assert_raise(ArgumentError) { eval(s("/\#{r}\xc2\xa1/s")) }
+    assert_raise(ArgumentError) { eval(s("/\xc2\xa1\#{r}/s")) }
+    assert_raise(ArgumentError) { eval(s("/\#{r}\xc2\xa1/s")) }
 
     r = eval(e("/\xc2\xa1/"))
-    #assert_raise(ArgumentError) { /\xc2\xa1#{r}/s }
+    assert_raise(ArgumentError) { /\xc2\xa1#{r}/s }
 
     r = /\xc2\xa1/e
-    #assert_raise(ArgumentError) { /\xc2\xa1#{r}/s }
+    assert_raise(ArgumentError) { /\xc2\xa1#{r}/s }
   end
 
   def test_begin_end_offset
@@ -560,7 +560,7 @@ class TestM17N < Test::Unit::TestCase
     }
     assert_regexp_fixed_ascii8bit(/#{}\xc2\xa1/n)
     assert_regexp_fixed_ascii8bit(/\xc2\xa1#{}/n)
-    #assert_raise(SyntaxError) { s1, s2 = s('\xc2'), s('\xa1'); /#{s1}#{s2}/ }
+    assert_nothing_raised { s1, s2 = a('\xc2'), a('\xa1'); /#{s1}#{s2}/ }
   end
 
   def test_dynamic_eucjp_regexp
@@ -570,7 +570,7 @@ class TestM17N < Test::Unit::TestCase
     assert_raise(SyntaxError) { eval('/\xc2#{}/e') }
     assert_raise(SyntaxError) { eval('/#{}\xc2/e') }
     assert_raise(SyntaxError) { eval('/\xc2#{}\xa1/e') }
-    #assert_raise(SyntaxError) { s1, s2 = e('\xc2'), e('\xa1'); /#{s1}#{s2}/ }
+    assert_raise(ArgumentError) { s1, s2 = e('\xc2'), e('\xa1'); /#{s1}#{s2}/ }
   end
 
   def test_dynamic_sjis_regexp
@@ -580,7 +580,7 @@ class TestM17N < Test::Unit::TestCase
     assert_raise(SyntaxError) { eval('/\x81#{}/s') }
     assert_raise(SyntaxError) { eval('/#{}\x81/s') }
     assert_raise(SyntaxError) { eval('/\x81#{}\xa1/s') }
-    #assert_raise(SyntaxError) { s1, s2 = s('\x81'), s('\xa1'); /#{s1}#{s2}/ }
+    assert_raise(ArgumentError) { s1, s2 = s('\x81'), s('\xa1'); /#{s1}#{s2}/ }
   end
 
   def test_dynamic_utf8_regexp
@@ -590,7 +590,7 @@ class TestM17N < Test::Unit::TestCase
     assert_raise(SyntaxError) { eval('/\xc2#{}/u') }
     assert_raise(SyntaxError) { eval('/#{}\xc2/u') }
     assert_raise(SyntaxError) { eval('/\xc2#{}\xa1/u') }
-    #assert_raise(SyntaxError) { s1, s2 = u('\xc2'), u('\xa1'); /#{s1}#{s2}/ }
+    assert_raise(ArgumentError) { s1, s2 = u('\xc2'), u('\xa1'); /#{s1}#{s2}/ }
   end
 
   def test_regexp_unicode
-- 
[田中 哲][たなか あきら][Tanaka Akira]