From: Nobuyoshi Nakada Date: 2010-03-18T07:05:59+09:00 Subject: [ruby-dev:40691] Re: [Feature #973] EncDet again なかだです。 At Wed, 17 Mar 2010 22:46:43 +0900, Yusuke Endoh wrote in [ruby-dev:40687]: > 2) File.open 方式で決定する > #2 の選択肢は実現可能性が検証されていない気がするので、「パッチを > 書いてみたら実は難しいことがわかった → 1.9.2 見送り」という危険が > あるかもしれません。 別に難しくはありません。 $ ./ruby -Eus-ascii -e 'ARGV.each{|file|open(file, "r:magic-comment"){|f|p [f.path, f.external_encoding]}}' version.h lib/rexml/rexml.rb lib/rubygems/package.rb ["version.h", #] ["lib/rexml/rexml.rb", #] ["lib/rubygems/package.rb", #] diff --git c/include/ruby/io.h i/include/ruby/io.h index e05a0f5..f067831 100644 --- c/include/ruby/io.h +++ i/include/ruby/io.h @@ -96,4 +96,5 @@ typedef struct rb_io_t { /* #define FMODE_PREP 0x00010000 */ #define FMODE_SETENC_BY_BOM 0x00100000 +#define FMODE_SETENC_BY_MAGIC_COMMENT 0x00200000 #define GetOpenFile(obj,fp) rb_io_check_closed((fp) = RFILE(rb_io_taint_check(obj))->fptr) diff --git c/io.c i/io.c index 60afd6c..60761f0 100644 --- c/io.c +++ i/io.c @@ -4125,4 +4125,6 @@ rb_io_ext_int_to_encs(rb_encoding *ext, rb_encoding *intern, rb_encoding **enc, } +#define is_magic_comment(str) (STRCASECMP(str, "magic-comment") == 0) + static void parse_mode_enc(const char *estr, rb_encoding **enc_p, rb_encoding **enc2_p) @@ -4166,5 +4168,5 @@ parse_mode_enc(const char *estr, rb_encoding **enc_p, rb_encoding **enc2_p) ext_enc = rb_enc_from_index(idx); else { - if (idx != -2) + if (idx != -2 && !is_magic_comment(estr)) rb_warn("Unsupported encoding %s ignored", estr); ext_enc = NULL; @@ -4337,6 +4339,11 @@ rb_io_extract_modeenc(VALUE *vmode_p, VALUE *vperm_p, VALUE opthash, has_enc = 1; parse_mode_enc(p+1, &enc, &enc2); - if (io_encname_bom_p(p+1, 0)) + if (io_encname_bom_p(p+1, 0)) { fmode |= FMODE_SETENC_BY_BOM; + p += 4; + } + if (is_magic_comment(p+1)) { + fmode |= FMODE_SETENC_BY_BOM | FMODE_SETENC_BY_MAGIC_COMMENT; + } } else { @@ -4605,10 +4612,44 @@ io_strip_bom(VALUE io) } -static void -io_set_encoding_by_bom(VALUE io) +int rb_magic_comment_encoding(const char *str, long len); + +static int +io_parse_encoding_comment(VALUE io) { - int idx = io_strip_bom(io); + VALUE line = rb_io_gets(io); + char *s; + long n; + if (NIL_P(line)) return 0; + s = RSTRING_PTR(line); + n = RSTRING_LEN(line); + if (n >= 2 && s[0] == '#' && s[1] == '!') { + VALUE shbang = line; + line = rb_io_gets(io); + if (NIL_P(line)) { + rb_io_ungetbyte(io, shbang); + return 0; + } + rb_io_ungetbyte(io, line); + s = RSTRING_PTR(line); + n = RSTRING_LEN(line); + line = shbang; + } + rb_io_ungetbyte(io, line); + while (n > 0 && (*s == ' ' || *s == '\t')) { + s++; + n--; + } + if (n <= 0 || *s != '#') return 0; + return rb_magic_comment_encoding(s, n); +} - if (idx) { +static void +io_guess_encoding(VALUE io, int fmode) +{ + int idx; + if (((fmode & FMODE_SETENC_BY_BOM) && + (idx = io_strip_bom(io)) != 0) || + ((fmode & FMODE_SETENC_BY_MAGIC_COMMENT) && + (idx = io_parse_encoding_comment(io)) != 0)) { rb_io_t *fptr; GetOpenFile(io, fptr); @@ -4638,5 +4679,5 @@ rb_file_open_generic(VALUE io, VALUE filename, int oflags, int fmode, convconfig fptr->fd = rb_sysopen(fptr->pathv, oflags, perm); io_check_tty(fptr); - if (fmode & FMODE_SETENC_BY_BOM) io_set_encoding_by_bom(io); + io_guess_encoding(io, fmode); return io; @@ -6396,5 +6437,5 @@ rb_io_initialize(int argc, VALUE *argv, VALUE io) fp->stdio_file = stderr; - if (fmode & FMODE_SETENC_BY_BOM) io_set_encoding_by_bom(io); + io_guess_encoding(io, fmode); return io; } diff --git c/parse.y i/parse.y index 340a825..a42c8f6 100644 --- c/parse.y +++ i/parse.y @@ -6248,13 +6248,15 @@ magic_comment_marker(const char *str, long len) } +typedef int rb_magic_comment_func(const char *name, long nlen, const char *value, long vlen, void *arg); + static int -parser_magic_comment(struct parser_params *parser, const char *str, long len) +parse_magic_comment(const char *str, long len, rb_magic_comment_func *func, void *arg) { - VALUE name = 0, val = 0; + VALUE name = 0; const char *beg, *end, *vbeg, *vend; #define str_copy(_s, _p, _n) ((_s) \ ? (rb_str_resize((_s), (_n)), \ MEMCPY(RSTRING_PTR(_s), (_p), char, (_n)), (_s)) \ - : ((_s) = STR_NEW((_p), (_n)))) + : ((_s) = rb_str_new((_p), (_n)))) if (len <= 7) return FALSE; @@ -6266,7 +6268,4 @@ parser_magic_comment(struct parser_params *parser, const char *str, long len) /* %r"([^\\s\'\":;]+)\\s*:\\s*(\"(?:\\\\.|[^\"])*\"|[^\"\\s;]+)[\\s;]*" */ while (len > 0) { -#ifndef RIPPER - const struct magic_comment *p = magic_comments; -#endif char *s; int i; @@ -6321,24 +6320,68 @@ parser_magic_comment(struct parser_params *parser, const char *str, long len) if (s[i] == '-') s[i] = '_'; } + if ((*func)(s, n, vbeg, vend - vbeg, arg)) break; + } + + return TRUE; +} + +static int +magic_comment_i(const char *name, long nlen, const char *value, long vlen, void *arg) +{ + struct parser_params *parser = arg; #ifndef RIPPER - do { - if (STRNCASECMP(p->name, s, n) == 0) { - n = vend - vbeg; - if (p->length) { - n = (*p->length)(parser, vbeg, n); - } - str_copy(val, vbeg, n); - (*p->func)(parser, s, RSTRING_PTR(val)); - break; + const struct magic_comment *p = magic_comments; + do { + if (STRNCASECMP(p->name, name, nlen) == 0) { + char *val; + if (p->length) { + vlen = (*p->length)(parser, value, vlen); } - } while (++p < magic_comments + numberof(magic_comments)); + val = ALLOCA_N(char, vlen + 1); + memcpy(val, value, vlen); + val[vlen] = '\0'; + (*p->func)(parser, name, val); + break; + } + } while (++p < magic_comments + numberof(magic_comments)); #else - dispatch2(magic_comment, name, val); + dispatch2(magic_comment, name, val); #endif - } + return FALSE; +} + +static int +parser_magic_comment(struct parser_params *parser, const char *str, long len) +{ + return parse_magic_comment(str, len, magic_comment_i, (void *)parser); +} + +static int +find_magic_comment_encoding(const char *name, long nlen, const char *value, long vlen, void *arg) +{ + char *val; + switch (nlen) { + case 8: + if (STRNCASECMP("en", name, 2) != 0) return FALSE; + name += 2; + case 6: + if (STRNCASECMP("coding", name, 6) != 0) return FALSE; + } + vlen = parser_encode_length(0, value, vlen); + memcpy(val = ALLOCA_N(char, vlen + 1), value, vlen); + val[vlen] = '\0'; + *(int *)arg = rb_enc_find_index(val); return TRUE; } +int +rb_magic_comment_encoding(const char *str, long len) +{ + int idx = 0; + if (!parse_magic_comment(str, len, find_magic_comment_encoding, &idx)) return 0; + return idx; +} + static void set_file_encoding(struct parser_params *parser, const char *str, const char *send) -- --- 僕の前にBugはない。 --- 僕の後ろにBugはできる。 中田 伸悦