From: David Flanagan Date: 2007-10-13T04:52:37+09:00 Subject: file encoding comments and a patch to parse.y This is a multi-part message in MIME format. --------------040305070904040203090208 Content-Type: text/plain; charset=ISO-8859-1; format=flowed Content-Transfer-Encoding: 7bit Matz, Nobu: I've just looked over your recent patches to parse.y for file encoding comments. set_file_encoding is a nice piece of work! What a sweet switch statement :-) I notice, however, that you use strncasecmp in one part of the function but only check for lowercase letters in the switch. This results in the strange behavior where this comment sets the encoding to UTF-8 #CODING: utf-8 But this comment, with just one extra space, does not: # CODING: utf-8 The attached file "patch1" is a simple patch that fixes this problem. The attached "patch2" is a larger patch that deletes all of the magic_comment code in the file because the new set_file_encoding function is general enough that it is no longer needed. I'm in over my head here and this may be a completely inappropriate patch, especially since it deletes some ifdef RIPPER stuff. But while I was making an aggressive patch like this, I took the liberty of changing the method name set_file_encoding to set_encoding_from_comment. I hope these are helpful. David --------------040305070904040203090208 Content-Type: text/plain; name="patch1" Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="patch1" Index: parse.y =================================================================== --- parse.y (revision 13687) +++ parse.y (working copy) @@ -5693,12 +5693,12 @@ for (;;) { if (send - str <= 6) return; switch (str[6]) { - case 'c': str += 6; continue; - case 'o': str += 5; continue; - case 'd': str += 4; continue; - case 'i': str += 3; continue; - case 'n': str += 2; continue; - case 'g': str += 1; continue; + case 'C': case 'c': str += 6; continue; + case 'O': case 'o': str += 5; continue; + case 'D': case 'd': str += 4; continue; + case 'I': case 'i': str += 3; continue; + case 'N': case 'n': str += 2; continue; + case 'G': case 'g': str += 1; continue; case '=': case ':': sep = 1; str += 6; --------------040305070904040203090208 Content-Type: text/plain; name="patch2" Content-Transfer-Encoding: 7bit Content-Disposition: inline; filename="patch2" Index: parse.y =================================================================== --- parse.y (revision 13687) +++ parse.y (working copy) @@ -5534,7 +5534,6 @@ #endif } -/* emacsen -*- hack */ static void parser_set_encode(struct parser_params *parser, const char *name) { @@ -5545,147 +5544,10 @@ parser->enc = rb_enc_from_index(idx); } -#ifndef RIPPER -typedef void (*rb_magic_comment_setter_t)(struct parser_params *parser, const char *name, const char *val); - static void -magic_comment_encoding(struct parser_params *parser, const char *name, const char *val) +set_encoding_from_comment(struct parser_params *parser, + const char *str, const char *send) { - if (parser && parser->line_count != (parser->has_shebang ? 2 : 1)) - return; - parser_set_encode(parser, val); -} - -struct magic_comment { - const char *name; - rb_magic_comment_setter_t func; -}; - -static const struct magic_comment magic_comments[] = { - {"coding", magic_comment_encoding}, - {"encoding", magic_comment_encoding}, -}; -#endif - -static const char * -magic_comment_marker(const char *str, int len) -{ - int i = 2; - - while (i < len) { - switch (str[i]) { - case '-': - if (str[i-1] == '*' && str[i-2] == '-') { - return str + i + 1; - } - i += 2; - break; - case '*': - if (i + 1 >= len) return 0; - if (str[i+1] != '-') { - i += 4; - } - else if (str[i-1] != '-') { - i += 2; - } - else { - return str + i + 2; - } - break; - default: - i += 3; - break; - } - } - return 0; -} - -static int -parser_magic_comment(struct parser_params *parser, const char *str, int len) -{ - VALUE name = 0, val = 0; - const char *beg, *end, *vbeg, *vend; -#define str_copy(_s, _p, _n) ((_s) \ - ? (rb_str_resize((_s), (_n)), \ - MEMCPY(RSTRING_PTR(_s), (_p), char, (_n)), (_s)) \ - : ((_s) = STR_NEW((_p), (_n)))) - - if (len <= 7) return Qfalse; - if (!(beg = magic_comment_marker(str, len))) return Qfalse; - if (!(end = magic_comment_marker(beg, str + len - beg))) return Qfalse; - str = beg; - len = end - beg - 3; - - /* %r"([^\\s\'\":;]+)\\s*:\\s*(\"(?:\\\\.|[^\"])*\"|[^\"\\s;]+)[\\s;]*" */ - while (len > 0) { -#ifndef RIPPER - const struct magic_comment *p = magic_comments; -#endif - int n = 0; - - for (; len > 0 && *str; str++, --len) { - switch (*str) { - case '\'': case '"': case ':': case ';': - continue; - } - if (!ISSPACE(*str)) break; - } - for (beg = str; len > 0; str++, --len) { - switch (*str) { - case '\'': case '"': case ':': case ';': - break; - default: - if (ISSPACE(*str)) break; - continue; - } - break; - } - for (end = str; len > 0 && ISSPACE(*str); str++, --len); - if (!len) break; - if (*str != ':') continue; - - do str++; while (--len > 0 && ISSPACE(*str)); - if (!len) break; - if (*str == '"') { - for (vbeg = ++str; --len > 0 && *str != '"'; str++) { - if (*str == '\\') { - --len; - ++str; - } - } - vend = str; - if (len) { - --len; - ++str; - } - } - else { - for (vbeg = str; len > 0 && *str != '"' && !ISSPACE(*str); --len, str++); - vend = str; - } - while (len > 0 && (*str == ';' || ISSPACE(*str))) --len, str++; - - n = end - beg; - str_copy(name, beg, n); -#ifndef RIPPER - do { - if (strncasecmp(p->name, RSTRING_PTR(name), n) == 0) { - str_copy(val, vbeg, vend - vbeg); - (*p->func)(parser, RSTRING_PTR(name), RSTRING_PTR(val)); - break; - } - } while (++p < magic_comments + sizeof(magic_comments) / sizeof(*p)); -#else - dispatch2(magic_comment, name, val); -#endif - } - - return Qtrue; -} - -static void -set_file_encoding(struct parser_params *parser, const char *str, const char *send) -{ int sep = 0; const char *beg = str; VALUE s; @@ -5693,12 +5555,12 @@ for (;;) { if (send - str <= 6) return; switch (str[6]) { - case 'c': str += 6; continue; - case 'o': str += 5; continue; - case 'd': str += 4; continue; - case 'i': str += 3; continue; - case 'n': str += 2; continue; - case 'g': str += 1; continue; + case 'C': case 'c': str += 6; continue; + case 'O': case 'o': str += 5; continue; + case 'D': case 'd': str += 4; continue; + case 'I': case 'i': str += 3; continue; + case 'N': case 'n': str += 2; continue; + case 'G': case 'g': str += 1; continue; case '=': case ':': sep = 1; str += 6; @@ -5815,13 +5677,8 @@ goto retry; case '#': /* it's a comment */ - if (!parser->has_shebang || parser->line_count != 1) { - /* no magic_comment in shebang line */ - if (!parser_magic_comment(parser, lex_p, lex_pend - lex_p)) { - if (parser->line_count == (parser->has_shebang ? 2 : 1)) { - set_file_encoding(parser, lex_p, lex_pend); - } - } + if (parser->line_count == (parser->has_shebang ? 2 : 1)) { + set_encoding_from_comment(parser, lex_p, lex_pend); } lex_p = lex_pend; #ifdef RIPPER --------------040305070904040203090208--