ruby-core

At 12:25 08/09/22, Michael Selig wrote:
>On Mon, 22 Sep 2008 12:35:49 +1000, Martin Duerst <duerst@it.aoyama.ac.jp>  
>wrote:
>
>>
>> Therefore, I think we should seriously consider this proposal,
>> and hopefully implement it before Sept. 25th. In terms of
>> implementation, I don't think it should be that difficult,
>> but it may be quite a bit of work to check
>> Encoding::default_internal in all the affected methods.
>
>Wow, that is rather ambitious - 3 days?

Well, that's the deadline for feature changes for 1.9.1.
It would be a real pity to wait for 2.0 for this.
The feature freeze wiki at
http://redmine.ruby-lang.org/wiki/ruby/DevelopersMeeting20080922
says that default_internal is currently pending, but that
this should be discussed/settled this week.

Anyhow, I had a look at the code, and it doesn't seem to be that
difficult. The function io_extract_encoding_option in io.c
seems to be central. I'm attaching a patch, which I hope is
a good start. I'm also writing to ruby-dev (in Japanese)
because that's where the real experts are.
The patch isn't as strict as your proposal with respect
to re-setting, but I'm fine either way.

I have tested this patch with code like the following
(called with -Eutf-8, -Eshift_jis, -Eeuc-jp, and without -E
option, in all combinations)

>>>>
Encoding.default_internal = 'utf-8'
      # tested with 'utf-8', 'shift_jis', and 'euc-jp'

s = "\u3042\u3044\u3046\u3048\u304A"
File.open('testout1.txt', 'w:shift_jis') do |f| f.write s end
File.open('testout2.txt', 'w:euc-jp') do |f| f.write s end
File.open('testout3.txt', 'w:utf-8') do |f| f.write s end

File.open('testout1.txt', 'r:shift_jis') do |f| s = f.read; p s.encoding end
File.open('testout2.txt', 'r:euc-jp') do |f| s = f.read; p s.encoding end
File.open('testout3.txt', 'r:utf-8') do |f| s = f.read; p s.encoding end
File.open('testout3.txt', 'r:ASCII-8BIT') do |f| s = f.read; p s.encoding end

# for next line, change file number to pick up default_internal
File.open('testout3.txt', 'r') do |f| s = f.read; p s.encoding end
>>>>

>The bulk of the implementation will be in the libraries, and I think many  
>of them need updating to cope with non-acsii encodings anyhow.

Yes. I'm not sure how libraries are affected by the feature
freeze, but they have to be fixed anyhow, completely independently
of default_internal. And I agree that this cannot be done in 3 days.

Regards,    Martin.

>> - We should think through various scenarios for output.
>>   I can't think of any problems just now, I just noticed
>>   the absence of considerations for output below.
>
>I did think about output to a certain extent, and one good thing is that  
>IO already seems to automatically transcode to the "external" encoding at  
>the moment. As for other classes, again I think most need updating to  
>support multiple encodings anyhow. They will at a minimum need a way of  
>having the user pass the "external" encoding (defaulting to  
>"default_external"), and do the transcode as necessary, based on the  
>encoding of the data to be output. However, as with IO, this behaviour  
>probably should happen no matter whether "default_internal" is implemented  
>or not.
>
>Cheers
>Mike
>

#-#-#  Martin J. Du"rst, Assoc. Professor, Aoyama Gakuin University
#-#-#  http://www.sw.it.aoyama.ac.jp       mailto:duerst@it.aoyama.ac.jp

Attachments (1)

patch_default_internal.txt (3 KB, text/x-diff)

Index: encoding.c
===================================================================
--- encoding.c	(revision 19510)
+++ encoding.c	(working copy)
@@ -1062,6 +1062,67 @@
 #endif
 }
 
+static int default_internal_index = -1;
+static rb_encoding *default_internal = 0;
+
+
+rb_encoding *
+rb_default_internal_encoding(void)
+{
+    return default_internal;
+}
+
+VALUE
+rb_enc_default_internal(void)
+{
+    return default_internal==0 ? Qnil : 
+	rb_enc_from_encoding(default_internal);
+}
+
+/*
+ * call-seq:
+ *   Encoding.default_internal => enc
+ *
+ * Returns default internal encoding (nil if unused).
+ *
+ */
+static VALUE
+get_default_internal(VALUE klass)
+{
+    return rb_enc_default_internal();
+}
+
+void
+rb_enc_set_default_internal(VALUE encoding)
+{
+    if (default_internal)
+	rb_warn("Resetting Encoding.default_internal");
+    if (encoding == Qnil) {
+        default_internal = 0;
+        default_internal_index = -1;
+    }
+    else {
+	default_internal = rb_to_encoding(encoding);
+	default_internal_index = rb_enc_to_index(default_internal);
+    }
+}
+
+/*
+ * call-seq:
+ *   Encoding.default_internal= enc => enc
+ *
+ * Sets default internal encoding (default is nil, i.e. unused).
+ * For use in main application; never use in a library!
+ * Returns nil. Produces a warning if reset.
+ *
+ */
+static VALUE
+set_default_internal(VALUE klass, VALUE encoding)
+{
+    rb_enc_set_default_internal(encoding);
+    return Qnil;
+}
+
 static void
 set_encoding_const(const char *name, rb_encoding *enc)
 {
@@ -1214,6 +1275,9 @@
     rb_define_singleton_method(rb_cEncoding, "default_external", get_default_external, 0);
     rb_define_singleton_method(rb_cEncoding, "locale_charmap", rb_locale_charmap, 0);
 
+    rb_define_singleton_method(rb_cEncoding, "default_internal",   get_default_internal, 0);
+    rb_define_singleton_method(rb_cEncoding, "default_internal=",  set_default_internal, 1);
+
     list = rb_ary_new2(enc_table.count);
     RBASIC(list)->klass = 0;
     rb_encoding_list = list;
Index: io.c
===================================================================
--- io.c	(revision 19510)
+++ io.c	(working copy)
@@ -3885,6 +3885,7 @@
     VALUE ecopts;
     int has_enc = 0, has_vmode = 0;
     VALUE intmode;
+    rb_encoding *def_internal;
 
     vmode = *vmode_p;
 
@@ -3972,6 +3973,20 @@
 
     *oflags_p = oflags;
     *fmode_p = fmode;
+    if (fmode&FMODE_READABLE && !enc2 && (def_internal=rb_default_internal_encoding())) {
+	rb_encoding *def_external = rb_default_external_encoding();
+	rb_encoding *ascii_8bit = rb_enc_find("ASCII-8BIT");
+	if (!enc) {
+	    if (def_external!=def_internal && def_external!=ascii_8bit) {
+	        enc  = def_internal;
+	        enc2 = def_external;
+	    }
+	}
+	else if (enc!=def_internal && enc!=ascii_8bit) {
+	    enc2 = enc;
+	    enc = def_internal;
+	}
+    }
     convconfig_p->enc = enc;
     convconfig_p->enc2 = enc2;
     convconfig_p->ecflags = ecflags;

Thread

Prev Next

In This Thread

Prev Next