From: Yohanes Santoso Date: 2006-01-31T07:42:51+09:00 Subject: PATCH: append option to sysread --=-=-= Hi, It is hard to do efficient incremental IO, a method where you just keep on buffering until you have a complete message in the buffer, in ruby because you end up creating many new objects. Here is a patch to IO#sysread that allows it to append to the given string instead of replacing it (1.8.x behaviour). Attached also the test script, test_io.rb, that simulates a pathological case of incremental IO. Basically the test would loop until it reads 10MB of data, on each loop it tries to detect for an end of message marker (which of course does not exist since this is a pathological case). Three methods are tested: 1. array method each return from io#sysread is appended into the array and then the array is joined and scanned for EOM. 2. concat method append the return value of io#sysread into a string and scan the string for EOM. 3. fastio method io#sysread appends read data into the given string and then scan the string for EOM. ~/tmp/ruby-1.8.4 $ ./ruby test_io.rb Reference content initialized. Size: 10485760 octets user system total real array 10M 1.550000 1.020000 2.570000 ( 2.784659) concat 10M 1.500000 1.080000 2.580000 ( 2.755901) fastio 10M 0.510000 0.040000 0.550000 ( 0.663247) YS. --=-=-= Content-Type: application/octet-stream Content-Disposition: attachment; filename=test_io.rb Content-Transfer-Encoding: base64 JDogPDwgJy4vLmV4dC9pNjg2LWxpbnV4JwokOiA8PCAnLi9saWInCnJlcXVpcmUgJ3NvY2tldCcK cmVxdWlyZSAnYmVuY2htYXJrJwoKUkVBRF9TSVpFID0gMTAqMTAyNCoxMDI0CgpkZWYgbWF0Y2hf Zm9yX25vbl9leGlzdGVudChzdHIpCiAgc3RyID1+IC9zZWFyY2hfZm9yX25vbl9leGlzdGVudF9l bmRfb2ZfbWVzc2FnZV9tYXJrZXIvCmVuZCAgCgoKJHJlZmVyZW5jZV9jb250ZW50ID0gbmlsCmRl ZiByZWZlcmVuY2VfY29udGVudAogIGlmIG5vdCAkcmVmZXJlbmNlX2NvbnRlbnQKICAgIGFyeSA9 IFtdCiAgICByZWFkX2xlbmd0aCA9IDAKICAgIFRDUFNvY2tldC5vcGVuKCdsb2NhbGhvc3QnLCAn Y2hhcmdlbicpIHt8c29ja3wgCiAgICAgIHdoaWxlIHJlYWRfbGVuZ3RoIDwgUkVBRF9TSVpFCiAg ICAgICAgYnVmZiA9IHNvY2suc3lzcmVhZChSRUFEX1NJWkUtcmVhZF9sZW5ndGgpIAogICAgICAg IGFyeSA8PCBidWZmCiAgICAgICAgcmVhZF9sZW5ndGggKz0gYnVmZi5sZW5ndGgKICAgICAgZW5k CiAgICB9CiAgICAkcmVmZXJlbmNlX2NvbnRlbnQgPSBhcnkuam9pbgogICAgd3JpdGVfdG9fZmls ZSgiKytyZWZlcmVuY2UiLCAkcmVmZXJlbmNlX2NvbnRlbnQpCiAgICBwdXRzICJSZWZlcmVuY2Ug Y29udGVudCBpbml0aWFsaXplZC4gU2l6ZTogI3skcmVmZXJlbmNlX2NvbnRlbnQuc2l6ZX0gb2N0 ZXRzIgogIGVuZAoKICAkcmVmZXJlbmNlX2NvbnRlbnQKZW5kCgpkZWYgYXNzZXJ0X2ludGVncml0 eShzdHIpCiAgcmFpc2UgIkludGVncml0eSBjb21wcm9taXNlZCIgaWYgc3RyICE9IHJlZmVyZW5j ZV9jb250ZW50CmVuZAoKZGVmIHdyaXRlX3RvX2ZpbGUoZm5hbWUsIHN0cikKICBGaWxlLm9wZW4o Zm5hbWUsICJ3Iikge3xmfAogICAgZi53cml0ZShzdHIpCiAgfQplbmQKCmRlZiBzaW11bGF0ZShi bSwgdGVzdF9uYW1lKQogIFRDUFNvY2tldC5vcGVuKCdsb2NhbGhvc3QnLCAnY2hhcmdlbicpIHt8 c29ja3wKICAgIHN0ciA9IG5pbAogICAgR0Muc3RhcnQKICAgIHNsZWVwKDUpICMgbGV0IG1lbW9y eSBzZXR0bGUgZG93biBmaXJzdAogICAgYm0ucmVwb3J0KCIje3Rlc3RfbmFtZX0gI3tSRUFEX1NJ WkUvMTAyNC8xMDI0fU0iKSB7CiAgICAgIHN0ciA9IHlpZWxkKHNvY2spCiAgICB9CiAgICB3cml0 ZV90b19maWxlKCIrKyN7dGVzdF9uYW1lfSIsIHN0cikKICAgIGFzc2VydF9pbnRlZ3JpdHkoc3Ry KQogIH0KZW5kCgoKZGVmIHNpbXVsYXRlX2luY3JlbWVudGFsX2lvCiAgQmVuY2htYXJrLmJtKCIg ICAgICAgICAgICAgICAgICAiLmxlbmd0aCkge3xibXwKICAgIHRvdGFsX2l0ZXIgPSB7fQogICAg CiAgICAKICAgICMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIwogICAgc2ltdWxhdGUoYm0s ICdhcnJheScpIHt8c29ja3wKICAgICAgYXJ5ID0gW10KICAgICAgc3RyID0gYXJ5LmpvaW4KICAg ICAgaXRlciA9IDAKICAgICAgd2hpbGUgc3RyLmxlbmd0aCA8IFJFQURfU0laRQogICAgICAgIGFy eSA8PCBzb2NrLnN5c3JlYWQoUkVBRF9TSVpFLXN0ci5sZW5ndGgpIAogICAgICAgIHN0ciA9IGFy eS5qb2luCiAgICAgICAgbWF0Y2hfZm9yX25vbl9leGlzdGVudChzdHIpCiAgICAgICAgaXRlciAr PSAxCiAgICAgIGVuZAogICAgICB0b3RhbF9pdGVyWzphcnJheV0gPSBpdGVyCiAgICAgIHN0cgog ICAgfQogICAgCiAgICAKCiAgICAjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMKICAgIHNp bXVsYXRlKGJtLCAnY29uY2F0Jykge3xzb2NrfAogICAgICBzdHIgPSAiIgogICAgICBpdGVyID0g MAogICAgICB3aGlsZSBzdHIubGVuZ3RoIDwgUkVBRF9TSVpFCiAgICAgICAgc3RyICs9IHNvY2su c3lzcmVhZChSRUFEX1NJWkUtc3RyLmxlbmd0aCkKICAgICAgICBtYXRjaF9mb3Jfbm9uX2V4aXN0 ZW50KHN0cikKICAgICAgICBpdGVyICs9MSAKICAgICAgZW5kCiAgICAgIHRvdGFsX2l0ZXJbOmNv bmNhdF0gPSBpdGVyCiAgICAgIHN0cgogICAgfQogICAgCgogICAgIyMjIyMjIyMjIyMjIyMjIyMj IyMjIyMjIyMjIyMjCiAgICBzaW11bGF0ZShibSwgJ2Zhc3RpbycpIHt8c29ja3wKICAgICAgc3Ry ID0gIiIKICAgICAgaXRlciA9IDAKICAgICAgd2hpbGUgc3RyLmxlbmd0aCA8IFJFQURfU0laRQog ICAgICAgIHNvY2suc3lzcmVhZChSRUFEX1NJWkUtc3RyLmxlbmd0aCwgc3RyLCB0cnVlKQogICAg ICAgIG1hdGNoX2Zvcl9ub25fZXhpc3RlbnQoc3RyKQogICAgICAgIGl0ZXIgKz0gMQogICAgICBl bmQKICAgICAgdG90YWxfaXRlcls6ZmFzdGlvXSA9IGl0ZXIKICAgICAgc3RyCiAgICB9CgogICAg IyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjCiAgICBwIHRvdGFsX2l0ZXIKICB9CmVuZAoK cmVmZXJlbmNlX2NvbnRlbnQKc2ltdWxhdGVfaW5jcmVtZW50YWxfaW8KCgoKCn4vdG1wL3J1Ynkt MS44LjQgJCAuL3J1YnkgdGVzdF9pby5yYiAKUmVmZXJlbmNlIGNvbnRlbnQgaW5pdGlhbGl6ZWQu IFNpemU6IDEwNDg1NzYwIG9jdGV0cwogICAgICAgICAgICAgICAgICAgICAgICB1c2VyICAgICBz eXN0ZW0gICAgICB0b3RhbCAgICAgICAgcmVhbAphcnJheSAxME0gICAgICAgICAgIDEuNTUwMDAw ICAgMS4wMjAwMDAgICAyLjU3MDAwMCAoICAyLjc4NDY1OSkKY29uY2F0IDEwTSAgICAgICAgICAx LjUwMDAwMCAgIDEuMDgwMDAwICAgMi41ODAwMDAgKCAgMi43NTU5MDEpCmZhc3RpbyAxME0gICAg ICAgICAgMC41MTAwMDAgICAwLjA0MDAwMCAgIDAuNTUwMDAwICggIDAuNjYzMjQ3KQp7OmNvbmNh dD0+MjA0LCA6YXJyYXk9PjIwMCwgOmZhc3Rpbz0+MTY3fQo= --=-=-= Content-Type: text/x-patch Content-Disposition: inline; filename=fastio.patch --- ruby-1.8.4.orig/io.c 2005-12-19 12:11:20.000000000 -0500 +++ ruby-1.8.4/io.c 2006-01-30 16:35:49.000000000 -0500 @@ -2362,38 +2362,53 @@ /* * call-seq: - * ios.sysread(integer ) => string + * ios.sysread(integer, string=nil, is_append=nil ) => string * * Reads integer bytes from ios using a low-level - * read and returns them as a string. Do not mix with other methods - * that read from ios or you may get unpredictable results. - * Raises SystemCallError on error and - * EOFError at end of file. + * read and returns them as a string. If string is provided, + * its content will be replaced or appended (if append is + * true). Do not mix with other methods that read from + * ios or you may get unpredictable results. Raises + * SystemCallError on error and EOFError at + * end of file. * * f = File.new("testfile") * f.sysread(16) #=> "This is line one" */ + static VALUE rb_io_sysread(argc, argv, io) int argc; VALUE *argv; VALUE io; { - VALUE len, str; + VALUE len, str, op_append; OpenFile *fptr; - long n, ilen; + long n, ilen, initial_len; + char *write_pos; - rb_scan_args(argc, argv, "11", &len, &str); + rb_scan_args(argc, argv, "12", &len, &str, &op_append); ilen = NUM2LONG(len); if (NIL_P(str)) { str = rb_str_new(0, ilen); - } - else { + initial_len = 0; + write_pos = RSTRING(str)->ptr; + } else { StringValue(str); rb_str_modify(str); - rb_str_resize(str, ilen); + initial_len = RSTRING(str)->len; + if ((NIL_P(op_append)) || + (op_append == Qfalse)) { + /* overwrites existing content */ + rb_str_resize(str, ilen); + write_pos = RSTRING(str)->ptr; + } else { + /* appends */ + rb_str_resize(str, RSTRING(str)->len + ilen); + write_pos = RSTRING(str)->ptr + initial_len; + } } if (ilen == 0) return str; @@ -2408,23 +2423,23 @@ n = fileno(fptr->f); rb_thread_wait_fd(fileno(fptr->f)); rb_io_check_closed(fptr); - if (RSTRING(str)->len != ilen) { + if (RSTRING(str)->len != initial_len + ilen) { rb_raise(rb_eRuntimeError, "buffer string modified"); } TRAP_BEG; - n = read(fileno(fptr->f), RSTRING(str)->ptr, ilen); + n = read(fileno(fptr->f), write_pos, ilen); TRAP_END; - + rb_str_unlocktmp(str); if (n == -1) { rb_sys_fail(fptr->path); } - rb_str_resize(str, n); + rb_str_resize(str, initial_len + n); if (n == 0 && ilen > 0) { rb_eof_error(); } - RSTRING(str)->len = n; - RSTRING(str)->ptr[n] = '\0'; + RSTRING(str)->len = initial_len + n; + RSTRING(str)->ptr[initial_len+n] = '\0'; OBJ_TAINT(str); return str; --=-=-=--