From: David Flanagan <david@...>
Date: 2007-12-22T08:46:39+09:00
Subject: io.c patch for two encodings in file mode

This is a multi-part message in MIME format.
--------------090403000605010503090808
Content-Type: text/plain; charset=ISO-8859-1; format=flowed
Content-Transfer-Encoding: 7bit

The attached patch adds support for filemodes like "r:utf-8:iso-8859-15" 
to io.c.  Works for reading and writing.  Also, it adds the 
external_encoding method to ARGF as well (and always returns the default 
external encoding for that stream).

This is only minimally tested, but I've verified that it can convert a 
Euro sign from ISO-8859-15 to UTF-8 and back for reading and writing.

To do the transcoding, I just rb_funcall to String.encode.  There might 
be more efficient ways to do it later.

I'm not confident that I've got everything done correctly, but I offer 
the patch in the hope that it helps with the last-minute rush to release 
1.9.1.

	David

--------------090403000605010503090808
Content-Type: text/plain;
 name="iopatch"
Content-Transfer-Encoding: 7bit
Content-Disposition: inline;
 filename="iopatch"

Index: include/ruby/io.h
===================================================================
--- include/ruby/io.h	(revision 14446)
+++ include/ruby/io.h	(working copy)
@@ -47,6 +47,7 @@
     int rbuf_capa;
     VALUE tied_io_for_writing;
     rb_encoding *enc;
+    rb_encoding *enc2;
 } rb_io_t;
 
 #define HAVE_RB_IO_T 1
@@ -91,6 +92,7 @@
     fp->rbuf_capa = 0;\
     fp->tied_io_for_writing = 0;\
     fp->enc = 0;\
+    fp->enc2 = 0;\
 } while (0)
 
 FILE *rb_io_stdio_file(rb_io_t *fptr);
Index: io.c
===================================================================
--- io.c	(revision 14446)
+++ io.c	(working copy)
@@ -124,7 +124,7 @@
 
 static VALUE argf;
 
-static ID id_write, id_read, id_getc, id_flush;
+static ID id_write, id_read, id_getc, id_flush, id_encode;
 
 extern char *ruby_inplace_mode;
 
@@ -622,6 +622,26 @@
 {
     long len, n, r, l, offset = 0;
 
+    /*
+     * If an external encoding was specified and it differs from
+     * the strings encoding then we must transcode before writing.
+     * We must also transcode if two encodings were specified
+     */
+    if (fptr->enc && (fptr->enc2 || fptr->enc != rb_enc_get(str))) {
+	/* transcode str before output */
+	/* the methods in transcode.c are static, so call indirectly */
+	/* Can't use encode! because puts writes a frozen newline */
+	if (fptr->enc2) {
+	    str = rb_funcall(str, id_encode, 2, 
+			     rb_enc_from_encoding(fptr->enc),
+			     rb_enc_from_encoding(fptr->enc2));
+	}
+	else {
+	    str = rb_funcall(str, id_encode, 1, 
+			     rb_enc_from_encoding(fptr->enc));
+	}
+    }
+
     len = RSTRING_LEN(str);
     if ((n = len) <= 0) return n;
     if (fptr->wbuf == NULL && !(fptr->mode & FMODE_SYNC)) {
@@ -1279,7 +1299,17 @@
 {
     OBJ_TAINT(str);
     if (fptr->enc) {
-	rb_enc_associate(str, fptr->enc);
+	if (fptr->enc2) {
+	    /* two encodings, so transcode from enc2 to enc */
+            /* the methods in transcode.c are static, so call indirectly */
+	    str = rb_funcall(str, id_encode, 2, 
+			     rb_enc_from_encoding(fptr->enc2),
+			     rb_enc_from_encoding(fptr->enc));
+	}
+	else {
+	    /* just one encoding, so associate it with the string */
+	    rb_enc_associate(str, fptr->enc);
+	}
     }
     return str;
 }
@@ -3063,7 +3093,8 @@
 rb_io_mode_enc(rb_io_t *fptr, const char *mode)
 {
     const char *p0, *p1;
-    int idx;
+    char *enc2name;
+    int idx, idx2;
     
     p0 = strrchr(mode, ':');
     if (p0) {
@@ -3071,11 +3102,26 @@
 	if (idx >= 0) {
 	    fptr->enc = rb_enc_from_index(idx);
 	}
-#if 0
+	else {
+	    rb_warn("Unsupported encoding %s ignored", p0+1);
+	}
 	p1 = strchr(mode, ':');
 	if (p1 < p0) {
+	    enc2name = ALLOCA_N(char, p0-p1);
+	    strncpy(enc2name, p1+1, p0-p1-1);
+	    enc2name[p0-p1-1] = '\0';
+	    idx2=rb_enc_find_index(enc2name);
+	    if (idx2 == idx) {
+		rb_warn("Ignoring internal encoding %s: it is identical to external encoding %s",
+			enc2name, p0+1);
+	    }
+	    else if (idx2 >= 0) {
+		fptr->enc2 = rb_enc_from_index(idx2);
+	    }
+	    else {
+		rb_warn("Unsupported encoding %s ignored", enc2name);
+	    }
 	}
-#endif
     }
     else if (!(fptr->mode & FMODE_BINMODE)) {
 	fptr->enc = rb_default_external_encoding();
@@ -5703,6 +5749,11 @@
     return rb_enc_from_encoding(fptr->enc);
 }
 
+static VALUE
+argf_external_encoding(void)
+{
+    return rb_enc_default_external();
+}
 
 static VALUE
 argf_tell(void)
@@ -6155,6 +6206,7 @@
     id_read = rb_intern("read");
     id_getc = rb_intern("getc");
     id_flush = rb_intern("flush");
+    id_encode = rb_intern("encode");
 
     rb_define_global_function("syscall", rb_f_syscall, -1);
 
@@ -6339,6 +6391,8 @@
     rb_define_singleton_method(argf, "lineno",   argf_lineno, 0);
     rb_define_singleton_method(argf, "lineno=",  argf_set_lineno, 1);
 
+    rb_define_singleton_method(argf, "external_encoding", argf_external_encoding, 0);
+
     rb_global_variable(&current_file);
     rb_define_readonly_variable("$FILENAME", &filename);
     filename = rb_str_new2("-");

--------------090403000605010503090808--