From: Robert Klemme <shortcutter@...>
Date: 2011-03-23T21:27:36+09:00
Subject: Re: A question about Ruby 1.9's "external encoding"

On Wed, Mar 23, 2011 at 12:59 PM, Albert Schlef <albertschlef@gmail.com> wrote:
> Robert K. wrote in post #988839:
>> What *I* find slightly puzzling is this:
>>
>> irb(main):001:0> s1 = "a"
>> => "a"
>> irb(main):002:0> s1.encoding
>> => #<Encoding:UTF-8>
>> irb(main):003:0> s2 = s1.encode 'ISO-8859-1'
>> => "a"
>> irb(main):004:0> s2.encoding
>> => #<Encoding:ISO-8859-1>
>> irb(main):005:0> s1 == s2
>> => true
>> irb(main):006:0> s1.eql? s2
>> => true
>> irb(main):007:0> [s1.hash, s2.hash]
>> => [1003075638, 1003075638]
>> irb(main):008:0> [s1.hash, s2.hash].uniq
>> => [1003075638]
>> irb(main):009:0> s1.encoding == s2.encoding
>> => false
>>
>> Apparently only the byte representation is used for equivalence checks
>> and the encoding is ignored.
>
> I don't think this is true:
>
> irb(main):043:0> utf = "\u05D0"  # Alef
> => "א"
> irb(main):044:0> latin = utf.dup; latin.force_encoding 'ISO-8859-1'
> => "�\x90"
> irb(main):045:0> [utf.bytes.to_a, latin.bytes.to_a]  # They have the
> same bytes
> => [[215, 144], [215, 144]]
> irb(main):048:0> [utf.valid_encoding?, latin.valid_encoding?] # And are
> ok
> => [true, true]
> irb(main):046:0> utf == latin   # But they aren't equal
> => false

Thanks for the interesting example!  I noticed:

irb(main):008:0> utf.length
=> 1
irb(main):009:0> latin.length
=> 2

> In your case it's good the strings are considered equal: we want to know
> if the letters are all the same. "a" is "a"... no matter what encoding.

Turns out the encoding is considered in comparison (read bottom up):

int
rb_str_comparable(VALUE str1, VALUE str2)
{
    int idx1, idx2;
    int rc1, rc2;

    if (RSTRING_LEN(str1) == 0) return TRUE;
    if (RSTRING_LEN(str2) == 0) return TRUE;
    idx1 = ENCODING_GET(str1);
    idx2 = ENCODING_GET(str2);
    if (idx1 == idx2) return TRUE;
    rc1 = rb_enc_str_coderange(str1);
    rc2 = rb_enc_str_coderange(str2);
    if (rc1 == ENC_CODERANGE_7BIT) {
        if (rc2 == ENC_CODERANGE_7BIT) return TRUE;
        if (rb_enc_asciicompat(rb_enc_from_index(idx2)))
            return TRUE;
    }
    if (rc2 == ENC_CODERANGE_7BIT) {
        if (rb_enc_asciicompat(rb_enc_from_index(idx1)))
            return TRUE;
    }
    return FALSE;
}

/* expect tail call optimization */
static VALUE
str_eql(const VALUE str1, const VALUE str2)
{
    const long len = RSTRING_LEN(str1);

    if (len != RSTRING_LEN(str2)) return Qfalse;
    if (!rb_str_comparable(str1, str2)) return Qfalse;
    if (memcmp(RSTRING_PTR(str1), RSTRING_PTR(str2), len) == 0)
        return Qtrue;
    return Qfalse;
}

VALUE
rb_str_equal(VALUE str1, VALUE str2)
{
    if (str1 == str2) return Qtrue;
    if (TYPE(str2) != T_STRING) {
        if (!rb_respond_to(str2, rb_intern("to_str"))) {
            return Qfalse;
        }
        return rb_equal(str2, str1);
    }
    return str_eql(str1, str2);
}

Now, everything is clear. ;-)

Cheers

robert

-- 
remember.guy do |as, often| as.you_can - without end
http://blog.rubybestpractices.com/