Re: Unicode: Progress?

Hi,

At Mon, 15 Oct 2007 20:18:03 +0900,
Yukihiro Matsumoto wrote in [ruby-core:12687]:
> |The knowledge about lower/upper case also seems to be included in 1.9's
> |Oniguruma, since /[[:upper:]]/u works as expected for me.
> 
> Oniguruma does case normalization.  Case conversion both way is more
> difficult.  I don't say it's impossible, but please allow us more
> time.

How about this patch?


Index: encoding.c
===================================================================
--- encoding.c	(revision 13717)
+++ encoding.c	(working copy)
@@ -461,4 +461,23 @@ rb_enc_tolower(int c, rb_encoding *enc)
 }
 
+int
+rb_enc_casefold(char *to, const char *p, const char *e, rb_encoding *enc)
+{
+    OnigCaseFoldCodeItem items[ONIGENC_GET_CASE_FOLD_CODES_MAX_NUM];
+    int i, n;
+
+    n = ONIGENC_GET_CASE_FOLD_CODES_BY_STR(enc, 0, (const UChar *)p, (const UChar *)e, items);
+    for (i = 0; i < n; ++i) {
+	if (items[i].code_len == 1) {
+	    rb_enc_mbcput(items[i].code[0], to, enc);
+	    return items[i].byte_len;
+	}
+    }
+    for (n = i = 0; i < items[0].code_len; ++i) {
+	n += rb_enc_mbcput(items[0].code[i], to + n, enc);
+    }
+    return items[0].byte_len;
+}
+
 static VALUE
 enc_to_s(VALUE self)
Index: string.c
===================================================================
--- string.c	(revision 13717)
+++ string.c	(working copy)
@@ -3048,4 +3071,68 @@ rb_str_dump(VALUE str)
 
 
+#define CHANGECASE_SETUP()					\
+    rb_encoding *enc;						\
+    char *s, *send, *t, *b;					\
+    long len;							\
+    int modify = 0, n, c, asciicompat;				\
+    VALUE str = *strp, str2 = str_new(0, 0, RSTRING_LEN(str));	\
+								\
+    *strp = str2;						\
+    enc = rb_enc_get(str);					\
+    asciicompat = rb_enc_asciicompat(enc);			\
+    rb_enc_associate(str2, enc);				\
+    s = RSTRING_PTR(str); send = RSTRING_END(str);		\
+    b = s;							\
+    len = 0
+
+#define CHANGECASE_FINISH()			\
+    if (s > b) {				\
+	MAKEROOM(s - b);			\
+    }						\
+    STR_SET_LEN(str2, len);			\
+    RSTRING_PTR(str2)[len] = 0;			\
+    if (modify) return str2;			\
+    return Qnil
+
+#define MAKEROOM(n) do {			\
+	RESIZE_CAPA(str2, len + (s - b) + (n)); \
+	t = RSTRING_PTR(str2);			\
+	MEMCPY(t + len, b, char, s - b);	\
+	len += s - b;				\
+    } while (0)
+
+#define SWAPCASE(c) ((c) ^ 0x20)
+
+#define CHANGECASE(ct)					\
+    c = rb_enc_codepoint(s, send, enc);			\
+    if (asciicompat && isascii(c) && ct(c)) {		\
+	MAKEROOM(1);					\
+	t[len++] = SWAPCASE(c);				\
+	b = ++s;					\
+	modify = 1;					\
+    }							\
+    else if (rb_enc_##ct(c, enc)) {			\
+	MAKEROOM(ONIGENC_CODE_TO_MBC_MAXLEN);		\
+	n = rb_enc_casefold(t + len, s, send, enc);	\
+	if (n > 0) len += n;				\
+	s += rb_enc_codelen(c, enc);			\
+	b = s;						\
+	modify = 1;					\
+    }							\
+    else {						\
+	s += rb_enc_codelen(c, enc);			\
+    }
+
+static VALUE
+str_upcase(VALUE *strp)
+{
+    CHANGECASE_SETUP();
+    while (s < send) {
+	CHANGECASE(islower);
+    }
+    CHANGECASE_FINISH();
+}
+
+
 /*
  *  call-seq:
@@ -3060,24 +3147,9 @@ static VALUE
 rb_str_upcase_bang(VALUE str)
 {
-    rb_encoding *enc;
-    char *s, *send;
-    int modify = 0;
-
+    VALUE str2 = str;
     rb_str_modify(str);
-    enc = rb_enc_get(str);
-    s = RSTRING_PTR(str); send = RSTRING_END(str);
-    while (s < send) {
-	int c = rb_enc_codepoint(s, send, enc);
-
-	if (rb_enc_islower(c, enc)) {
-	    /* assuming toupper returns codepoint with same size */
-	    rb_enc_mbcput(rb_enc_toupper(c, enc), s, enc);
-	    modify = 1;
-	}
-	s += rb_enc_codelen(c, enc);
-    }
-
-    if (modify) return str;
-    return Qnil;
+    if (NIL_P(str_upcase(&str2))) return Qnil;
+    rb_str_replace(str, str2);
+    return str;
 }
 
@@ -3098,10 +3170,22 @@ static VALUE
 rb_str_upcase(VALUE str)
 {
-    str = rb_str_dup(str);
-    rb_str_upcase_bang(str);
+    VALUE klass = RBASIC(str)->klass;
+    str_upcase(&str);
+    RBASIC(str)->klass = klass;
     return str;
 }
 
 
+static VALUE
+str_downcase(VALUE *strp)
+{
+    CHANGECASE_SETUP();
+    while (s < send) {
+	CHANGECASE(isupper);
+    }
+    CHANGECASE_FINISH();
+}
+
+
 /*
  *  call-seq:
@@ -3116,24 +3200,9 @@ static VALUE
 rb_str_downcase_bang(VALUE str)
 {
-    rb_encoding *enc;
-    char *s, *send;
-    int modify = 0;
-
+    VALUE str2 = str;
     rb_str_modify(str);
-    enc = rb_enc_get(str);
-    s = RSTRING_PTR(str); send = RSTRING_END(str);
-    while (s < send) {
-	int c = rb_enc_codepoint(s, send, enc);
-
-	if (rb_enc_isupper(c, enc)) {
-	    /* assuming toupper returns codepoint with same size */
-	    rb_enc_mbcput(rb_enc_tolower(c, enc), s, enc);
-	    modify = 1;
-	}
-	s += rb_enc_codelen(c, enc);
-    }
-
-    if (modify) return str;
-    return Qnil;
+    if (NIL_P(str_downcase(&str2))) return Qnil;
+    rb_str_replace(str, str2);
+    return str;
 }
 
@@ -3154,10 +3223,23 @@ static VALUE
 rb_str_downcase(VALUE str)
 {
-    str = rb_str_dup(str);
-    rb_str_downcase_bang(str);
+    VALUE klass = RBASIC(str)->klass;
+    str_downcase(&str);
+    RBASIC(str)->klass = klass;
     return str;
 }
 
 
+static VALUE
+str_capitalize(VALUE *strp)
+{
+    CHANGECASE_SETUP();
+    CHANGECASE(islower);
+    while (s < send) {
+	CHANGECASE(isupper);
+    }
+    CHANGECASE_FINISH();
+}
+
+
 /*
  *  call-seq:
@@ -3177,30 +3259,9 @@ static VALUE
 rb_str_capitalize_bang(VALUE str)
 {
-    rb_encoding *enc;
-    char *s, *send;
-    int modify = 0;
-    int c;
-
+    VALUE str2 = str;
     rb_str_modify(str);
-    enc = rb_enc_get(str);
-    if (RSTRING_LEN(str) == 0 || !RSTRING_PTR(str)) return Qnil;
-    s = RSTRING_PTR(str); send = RSTRING_END(str);
-
-    c = rb_enc_codepoint(s, send, enc);
-    if (rb_enc_islower(c, enc)) {
-	rb_enc_mbcput(rb_enc_toupper(c, enc), s, enc);
-	modify = 1;
-    }
-    s += rb_enc_codelen(c, enc);
-    while (s < send) {
-	c = rb_enc_codepoint(s, send, enc);
-	if (rb_enc_isupper(c, enc)) {
-	    rb_enc_mbcput(rb_enc_tolower(c, enc), s, enc);
-	    modify = 1;
-	}
-	s += rb_enc_codelen(c, enc);
-    }
-    if (modify) return str;
-    return Qnil;
+    if (NIL_P(str_capitalize(&str2))) return Qnil;
+    rb_str_replace(str, str2);
+    return str;
 }
 
@@ -3222,10 +3283,22 @@ static VALUE
 rb_str_capitalize(VALUE str)
 {
-    str = rb_str_dup(str);
-    rb_str_capitalize_bang(str);
+    VALUE klass = RBASIC(str)->klass;
+    str_capitalize(&str);
+    RBASIC(str)->klass = klass;
     return str;
 }
 
 
+static VALUE
+str_swapcase(VALUE *strp)
+{
+    CHANGECASE_SETUP();
+    while (s < send) {
+	CHANGECASE(isalpha);
+    }
+    CHANGECASE_FINISH();
+}
+
+
 /*
  *  call-seq:
@@ -3240,29 +3313,9 @@ static VALUE
 rb_str_swapcase_bang(VALUE str)
 {
-    rb_encoding *enc;
-    char *s, *send;
-    int modify = 0;
-
+    VALUE str2 = str;
     rb_str_modify(str);
-    enc = rb_enc_get(str);
-    s = RSTRING_PTR(str); send = RSTRING_END(str);
-    while (s < send) {
-	int c = rb_enc_codepoint(s, send, enc);
-
-	if (rb_enc_isupper(c, enc)) {
-	    /* assuming toupper returns codepoint with same size */
-	    rb_enc_mbcput(rb_enc_tolower(c, enc), s, enc);
-	    modify = 1;
-	}
-	else if (rb_enc_islower(c, enc)) {
-	    /* assuming toupper returns codepoint with same size */
-	    rb_enc_mbcput(rb_enc_toupper(c, enc), s, enc);
-	    modify = 1;
-	}
-	s += rb_enc_codelen(c, enc);
-    }
-
-    if (modify) return str;
-    return Qnil;
+    if (NIL_P(str_swapcase(&str2))) return Qnil;
+    rb_str_replace(str, str2);
+    return str;
 }
 
@@ -3283,6 +3336,7 @@ static VALUE
 rb_str_swapcase(VALUE str)
 {
-    str = rb_str_dup(str);
-    rb_str_swapcase_bang(str);
+    VALUE klass = RBASIC(str)->klass;
+    str_swapcase(&str);
+    RBASIC(str)->klass = klass;
     return str;
 }
Index: include/ruby/encoding.h
===================================================================
--- include/ruby/encoding.h	(revision 13717)
+++ include/ruby/encoding.h	(working copy)
@@ -95,4 +95,5 @@ int rb_enc_codelen(int, rb_encoding*);
 #define rb_enc_asciicompat(enc) (rb_enc_mbminlen(enc)==1)
 
+int rb_enc_casefold(char *to, const char *p, const char *e, rb_encoding *enc);
 int rb_enc_toupper(int c, rb_encoding *enc);
 int rb_enc_tolower(int c, rb_encoding *enc);
Index: test/ruby/test_encoding.rb
===================================================================
--- test/ruby/test_encoding.rb	(revision 13717)
+++ test/ruby/test_encoding.rb	(working copy)
@@ -26,3 +26,66 @@ class TestEncoding < Test::Unit::TestCas
     end    
   end
+
+  # lettercase
+  UPSTR = "R\303\234BY".force_encoding("utf-8")
+  LOWSTR = "r\303\274by".force_encoding("utf-8")
+  CAPSTR = "R\303\274by".force_encoding("utf-8")
+  MIXSTR = "r\303\234By".force_encoding("utf-8")
+  MIX2STR = "R\303\274bY".force_encoding("utf-8")
+
+  def test_upcase
+    assert_equal(UPSTR, UPSTR.upcase)
+    assert_equal(UPSTR, LOWSTR.upcase)
+    assert_equal(UPSTR, CAPSTR.upcase)
+    assert_equal(UPSTR, MIXSTR.upcase)
+  end
+
+  def test_downcase
+    assert_equal(LOWSTR, UPSTR.downcase)
+    assert_equal(LOWSTR, LOWSTR.downcase)
+    assert_equal(LOWSTR, CAPSTR.downcase)
+    assert_equal(LOWSTR, MIXSTR.downcase)
+  end
+
+  def test_capitalize
+    assert_equal(CAPSTR, UPSTR.capitalize)
+    assert_equal(CAPSTR, LOWSTR.capitalize)
+    assert_equal(CAPSTR, CAPSTR.capitalize)
+    assert_equal(CAPSTR, MIXSTR.capitalize)
+  end
+
+  def test_swapcase
+    assert_equal(LOWSTR, UPSTR.swapcase)
+    assert_equal(UPSTR, LOWSTR.swapcase)
+    assert_equal(MIX2STR, MIXSTR.swapcase)
+    assert_equal(MIXSTR, MIX2STR.swapcase)
+  end
+
+  def test_upcase!
+    assert_nil(UPSTR.dup.upcase!)
+    assert_equal(UPSTR, LOWSTR.dup.upcase!)
+    assert_equal(UPSTR, CAPSTR.dup.upcase!)
+    assert_equal(UPSTR, MIXSTR.dup.upcase!)
+  end
+
+  def test_downcase!
+    assert_equal(LOWSTR, UPSTR.dup.downcase!)
+    assert_nil(LOWSTR.dup.downcase!)
+    assert_equal(LOWSTR, CAPSTR.dup.downcase!)
+    assert_equal(LOWSTR, MIXSTR.dup.downcase!)
+  end
+
+  def test_capitalize!
+    assert_equal(CAPSTR, UPSTR.dup.capitalize!)
+    assert_equal(CAPSTR, LOWSTR.dup.capitalize!)
+    assert_nil(CAPSTR.dup.capitalize!)
+    assert_equal(CAPSTR, MIXSTR.dup.capitalize!)
+  end
+
+  def test_swapcase!
+    assert_equal(LOWSTR, UPSTR.dup.swapcase!)
+    assert_equal(UPSTR, LOWSTR.dup.swapcase!)
+    assert_equal(MIX2STR, MIXSTR.dup.swapcase!)
+    assert_equal(MIXSTR, MIX2STR.dup.swapcase!)
+  end
 end


-- 
Nobu Nakada