Re: Encoding Pragma

Hi,

At Fri, 13 May 2005 00:44:06 +0900,
Yukihiro Matsumoto wrote in [ruby-core:04909]:
> |I'm wondering how "at the top" can be detect.  ruby_sourceline
> |is defined differently between parse.y and ripper.
> 
> Aoki-san will take care of ripper.  ;-)
> Ripper may work with parser->parser_ruby_sourceline.

OK, this patch seems to work.


* intern.h (ruby_pragma): prototype.  [ruby-core:04881]

* parse.y (parser_pragma): parse Emacsen hack.

* parse.y (parser_prepare): deal with specific syntax at the top.

* parse.y (parser_yylex), ruby.c (load_file): support pragma.


Index: intern.h
===================================================================
RCS file: /cvs/ruby/src/ruby/intern.h,v
retrieving revision 1.166
diff -U2 -p -r1.166 intern.h
--- intern.h	19 Apr 2005 14:02:48 -0000	1.166
+++ intern.h	12 May 2005 04:39:48 -0000
@@ -395,4 +395,5 @@ void ruby_load_script _((void));
 void ruby_init_loadpath _((void));
 void ruby_incpush _((const char*));
+int ruby_pragma _((const char*, int));
 /* signal.c */
 VALUE rb_f_kill _((int, VALUE*));
Index: parse.y
===================================================================
RCS file: /cvs/ruby/src/ruby/parse.y,v
retrieving revision 1.378
diff -U2 -p -r1.378 parse.y
--- parse.y	1 May 2005 00:15:25 -0000	1.378
+++ parse.y	12 May 2005 16:51:05 -0000
@@ -165,4 +165,6 @@ struct parser_params {
     int toplevel_p;
 #endif
+    int line_count;
+    int has_shebang;
 };
 
@@ -4432,4 +4434,6 @@ parser_yyerror(parser, msg)
 }
 
+static void parser_prepare _((struct parser_params *parser));
+
 #ifndef RIPPER
 int ruby__end__seen;
@@ -4445,4 +4449,5 @@ yycompile(parser, f, line)
     NODE *node = 0;
     struct RVarmap *vp, *vars = ruby_dyna_vars;
+    const char *kcode_save;
 
     if (!compile_for_eval && rb_safe_level() == 0 &&
@@ -4461,17 +4466,20 @@ yycompile(parser, f, line)
 	if (line > 1) {
 	    VALUE str = rb_str_new(0,0);
-	    while (line > 1) {
+	    n = line - 1;
+	    do {
 		rb_ary_push(ruby_debug_lines, str);
-		line--;
-	    }
+	    } while (--n);
 	}
     }
 
+    kcode_save = rb_get_kcode();
     ruby_current_node = 0;
     ruby_sourcefile = rb_source_filename(f);
     ruby_sourceline = line - 1;
+    parser_prepare(parser);
     n = yyparse((void*)parser);
     ruby_debug_lines = 0;
     compile_for_eval = 0;
+    rb_set_kcode(kcode_save);
 
     vp = ruby_dyna_vars;
@@ -4622,4 +4630,5 @@ parser_nextc(parser)
 	    }
 	    ruby_sourceline++;
+	    parser->line_count++;
 	    lex_pbeg = lex_p = RSTRING(v)->ptr;
 	    lex_pend = lex_p + RSTRING(v)->len;
@@ -5338,4 +5347,170 @@ lvar_defined_gen(parser, id)
 }
 
+/* emacsen -*- hack */
+typedef void (*rb_pragma_setter_t) _((struct parser_params *parser, const char *name, const char *val));
+
+static void
+pragma_encoding(parser, name, val)
+    struct parser_params *parser;
+    const char *name, *val;
+{
+    if (parser && parser->line_count != (parser->has_shebang ? 2 : 1))
+	return;
+    rb_set_kcode(val);
+}
+
+struct pragma {
+    const char *name;
+    rb_pragma_setter_t func;
+};
+
+static const struct pragma pragmas[] = {
+    {"coding", pragma_encoding},
+};
+
+static const char *
+pragma_marker(str, len)
+    const char *str;
+    int len;
+{
+    int i = 2;
+
+    while (i < len) {
+	switch (str[i]) {
+	  case '-':
+	    if (str[i-1] == '*' && str[i-2] == '-') {
+		return str + i + 1;
+	    }
+	    i += 2;
+	    break;
+	  case '*':
+	    if (i + 1 >= len) return 0;
+	    if (str[i+1] != '-') {
+		i += 4;
+	    }
+	    else if (str[i-1] != '-') {
+		i += 2;
+	    }
+	    else {
+		return str + i + 2;
+	    }
+	    break;
+	  default:
+	    i += 3;
+	    break;
+	}
+    }
+    return 0;
+}
+
+static int
+parser_pragma(parser, str, len)
+    struct parser_params *parser;
+    const char *str;
+    int len;
+{
+    VALUE name = 0, val = 0;
+    const char *beg, *end, *vbeg, *vend;
+#define str_copy(_s, _p, _n) ((_s) \
+	? (rb_str_resize((_s), (_n)), \
+	   MEMCPY(RSTRING(_s)->ptr, (_p), char, (_n)), (_s)) \
+	: ((_s) = rb_str_new((_p), (_n))))
+
+    if (!(beg = pragma_marker(str, len))) return Qfalse;
+    if (!(end = pragma_marker(beg, str + len - beg))) return Qfalse;
+    str = beg;
+    len = end - beg - 3;
+    
+    /* %r"([^\\s\'\":;]+)\\s*:\\s*(\"(?:\\\\.|[^\"])*\"|[^\"\\s;]+)[\\s;]*" */
+    while (len > 0) {
+	const struct pragma *p = pragmas;
+	int n = 0;
+
+	for (; len > 0 && *str; str++, --len) {
+	    switch (*str) {
+	      case '\'': case '"': case ':': case ';':
+		continue;
+	    }
+	    if (!ISSPACE(*str)) break;
+	}
+	for (beg = str; len > 0; str++, --len) {
+	    switch (*str) {
+	      case '\'': case '"': case ':': case ';':
+		break;
+	      default:
+		if (ISSPACE(*str)) break;
+		continue;
+	    }
+	    break;
+	}
+	for (end = str; len > 0 && ISSPACE(*str); str++, --len);
+	if (!len) break;
+	if (*str != ':') continue;
+
+	do str++; while (--len > 0 && ISSPACE(*str));
+	if (!len) break;
+	if (*str == '"') {
+	    for (vbeg = ++str; --len > 0 && *str != '"'; str++) {
+		if (*str == '\\') {
+		    --len;
+		    ++str;
+		}
+	    }
+	    vend = str;
+	    if (len) {
+		--len;
+		++str;
+	    }
+	}
+	else {
+	    for (vbeg = str; len > 0 && *str != '"' && !ISSPACE(*str); --len, str++);
+	    vend = str;
+	}
+	while (len > 0 && (*str == ';' || ISSPACE(*str))) --len, str++;
+
+	str_copy(name, beg, end - beg);
+	rb_funcall(name, rb_intern("downcase!"), 0);
+	do {
+	    if (strncmp(p->name, RSTRING(name)->ptr, n) == 0) {
+		str_copy(val, vbeg, vend - vbeg);
+		(*p->func)(parser, RSTRING(name)->ptr, RSTRING(val)->ptr);
+		break;
+	    }
+	} while (++p < pragmas + sizeof(pragmas) / sizeof(*p));
+    }
+
+    return Qtrue;
+}
+
+int
+ruby_pragma(str, len)
+    const char *str;
+    int len;
+{
+    return parser_pragma(0, str, len);
+}
+
+static void
+parser_prepare(parser)
+    struct parser_params *parser;
+{
+    int c = nextc();
+    switch (c) {
+      case '#':
+	if (peek('!')) parser->has_shebang = 1;
+	break;
+      case 0xef:		/* UTF-8 BOM marker */
+	if (ruby_sourceline == 1 && lex_pend - lex_p >= 2 &&
+	    (unsigned char)lex_p[0] == 0xbb &&
+	    (unsigned char)lex_p[1] == 0xbf) {
+	    rb_set_kcode("UTF-8");
+	}
+	break;
+      case EOF:
+	return;
+    }
+    pushback(c);
+}
+
 #define IS_ARG() (lex_state == EXPR_ARG || lex_state == EXPR_CMDARG)
 #define IS_BEG() (lex_state == EXPR_BEG || lex_state == EXPR_MID || lex_state == EXPR_VALUE || lex_state == EXPR_CLASS)
@@ -5404,12 +5579,9 @@ parser_yylex(parser)
 
       case '#':		/* it's a comment */
-	while ((c = nextc()) != '\n') {
-	    if (c == -1) {
-#ifdef RIPPER
-                ripper_dispatch_scan_event(parser, tCOMMENT);
-#endif
-		return 0;
-            }
+	if (!parser->has_shebang || parser->line_count != 1) {
+	    /* no pragma in shebang line */
+	    parser_pragma(parser, lex_p, lex_pend - lex_p);
 	}
+	lex_p = lex_pend;
 #ifdef RIPPER
         ripper_dispatch_scan_event(parser, tCOMMENT);
@@ -8825,4 +8997,5 @@ ripper_parse0(parser_v)
 
     Data_Get_Struct(parser_v, struct parser_params, parser);
+    parser_prepare(parser);
     ripper_yyparse((void*)parser);
     return parser->result;
Index: ruby.c
===================================================================
RCS file: /cvs/ruby/src/ruby/ruby.c,v
retrieving revision 1.100
diff -U2 -p -r1.100 ruby.c
--- ruby.c	1 May 2005 00:13:34 -0000	1.100
+++ ruby.c	12 May 2005 16:54:09 -0000
@@ -967,4 +967,9 @@ load_file(fname, script)
 		}
 	    }
+	    else if (RSTRING(line)->len > 7) {
+		ruby_pragma(RSTRING(line)->ptr, RSTRING(line)->len);
+		--line_start;
+		rb_io_ungetc(f, INT2FIX('\n'));
+	    }
 	}
 	else if (!NIL_P(c)) {


-- 
Nobu Nakada