From: Perry Smith Date: 2008-02-11T08:46:11+09:00 Subject: Changes to Ripper --Apple-Mail-2-1041676122 Content-Type: text/plain; charset=US-ASCII; format=flowed; delsp=yes Content-Transfer-Encoding: 7bit Hi, This is my first attempt at this so I do not know the protocol. I am trying to use Ripper to do a couple of things. One is to develop something similar to cscope and another is to get better coloring inside of emacs. In both of these tasks, I need good parsing of Ruby. I discovered Ripper and I'm trying to use it. The problem is that I would like to have the beginning and ending terminals included in the Ripper output. For example, currently a string is just the contents of the string. It would be nice to have the start and end so that coloring the text in emacs would be simpler. Also, the start and end of a method -- the "def" and "end" keywords are not in the Ripper output. These changes demonstrate what I am talking about. They are not complete but I altered only about five to eight entities. I wanted to see if this type of change would be accepted before I spent too much time on it. Changelog: parser.y (ripper specific code) Changes to provide start and end terminals in parsing constructs. --Apple-Mail-2-1041676122 Content-Disposition: attachment; filename=ripper.diff Content-Type: application/octet-stream; x-unix-mode=0664; name="ripper.diff" Content-Transfer-Encoding: 7bit Index: parse.y =================================================================== --- parse.y (revision 15424) +++ parse.y (working copy) @@ -487,13 +487,21 @@ static VALUE ripper_dispatch3(struct parser_params*,ID,VALUE,VALUE,VALUE); static VALUE ripper_dispatch4(struct parser_params*,ID,VALUE,VALUE,VALUE,VALUE); static VALUE ripper_dispatch5(struct parser_params*,ID,VALUE,VALUE,VALUE,VALUE,VALUE); +static VALUE ripper_dispatch6(struct parser_params*,ID,VALUE,VALUE,VALUE,VALUE,VALUE,VALUE); +static VALUE ripper_dispatch7(struct parser_params*,ID,VALUE,VALUE,VALUE,VALUE,VALUE,VALUE,VALUE); +static VALUE ripper_dispatch8(struct parser_params*,ID,VALUE,VALUE,VALUE,VALUE,VALUE,VALUE,VALUE,VALUE); +static VALUE ripper_dispatch9(struct parser_params*,ID,VALUE,VALUE,VALUE,VALUE,VALUE,VALUE,VALUE,VALUE,VALUE); -#define dispatch0(n) ripper_dispatch0(parser, TOKEN_PASTE(ripper_id_, n)) -#define dispatch1(n,a) ripper_dispatch1(parser, TOKEN_PASTE(ripper_id_, n), a) -#define dispatch2(n,a,b) ripper_dispatch2(parser, TOKEN_PASTE(ripper_id_, n), a, b) -#define dispatch3(n,a,b,c) ripper_dispatch3(parser, TOKEN_PASTE(ripper_id_, n), a, b, c) -#define dispatch4(n,a,b,c,d) ripper_dispatch4(parser, TOKEN_PASTE(ripper_id_, n), a, b, c, d) -#define dispatch5(n,a,b,c,d,e) ripper_dispatch5(parser, TOKEN_PASTE(ripper_id_, n), a, b, c, d, e) +#define dispatch0(n) ripper_dispatch0(parser, TOKEN_PASTE(ripper_id_, n)) +#define dispatch1(n,a) ripper_dispatch1(parser, TOKEN_PASTE(ripper_id_, n), a) +#define dispatch2(n,a,b) ripper_dispatch2(parser, TOKEN_PASTE(ripper_id_, n), a, b) +#define dispatch3(n,a,b,c) ripper_dispatch3(parser, TOKEN_PASTE(ripper_id_, n), a, b, c) +#define dispatch4(n,a,b,c,d) ripper_dispatch4(parser, TOKEN_PASTE(ripper_id_, n), a, b, c, d) +#define dispatch5(n,a,b,c,d,e) ripper_dispatch5(parser, TOKEN_PASTE(ripper_id_, n), a, b, c, d, e) +#define dispatch6(n,a,b,c,d,e,f) ripper_dispatch6(parser, TOKEN_PASTE(ripper_id_, n), a, b, c, d, e, f) +#define dispatch7(n,a,b,c,d,e,f,g) ripper_dispatch7(parser, TOKEN_PASTE(ripper_id_, n), a, b, c, d, e, f, g) +#define dispatch8(n,a,b,c,d,e,f,g,h) ripper_dispatch8(parser, TOKEN_PASTE(ripper_id_, n), a, b, c, d, e, f, g, h) +#define dispatch9(n,a,b,c,d,e,f,g,h,i) ripper_dispatch9(parser, TOKEN_PASTE(ripper_id_, n), a, b, c, d, e, f, g, h, i) #define yyparse ripper_yyparse @@ -659,6 +667,7 @@ %type words qwords word_list qword_list word %type literal numeric dsym cpath %type bodystmt compstmt stmts stmt expr arg primary command command_call method_call +%type rparen rbracket %type expr_value arg_value primary_value %type if_tail opt_else case_body cases opt_rescue exc_list exc_var opt_ensure %type args call_args call_args2 opt_call_args @@ -678,36 +687,46 @@ /*% %type program reswords then do dot_or_colon %*/ -%token tUPLUS /* unary+ */ -%token tUMINUS /* unary- */ -%token tPOW /* ** */ -%token tCMP /* <=> */ -%token tEQ /* == */ -%token tEQQ /* === */ -%token tNEQ /* != */ -%token tGEQ /* >= */ -%token tLEQ /* <= */ -%token tANDOP tOROP /* && and || */ -%token tMATCH tNMATCH /* =~ and !~ */ -%token tDOT2 tDOT3 /* .. and ... */ -%token tAREF tASET /* [] and []= */ -%token tLSHFT tRSHFT /* << and >> */ -%token tCOLON2 /* :: */ -%token tCOLON3 /* :: at EXPR_BEG */ %token tOP_ASGN /* +=, -= etc. */ -%token tASSOC /* => */ -%token tLPAREN /* ( */ -%token tLPAREN_ARG /* ( */ -%token tRPAREN /* ) */ -%token tLBRACK /* [ */ -%token tLBRACE /* { */ -%token tLBRACE_ARG /* { */ -%token tSTAR /* * */ -%token tAMPER /* & */ -%token tLAMBDA /* -> */ -%token tSYMBEG tSTRING_BEG tXSTRING_BEG tREGEXP_BEG tWORDS_BEG tQWORDS_BEG -%token tSTRING_DBEG tSTRING_DVAR tSTRING_END tLAMBEG + /*%%%*/ +%token + /*% +%token +%*/ + tUPLUS /* unary+ */ + tUMINUS /* unary- */ + tPOW /* ** */ + tCMP /* <=> */ + tEQ /* == */ + tEQQ /* === */ + tNEQ /* != */ + tGEQ /* >= */ + tLEQ /* <= */ + tANDOP tOROP /* && and || */ + tMATCH tNMATCH /* =~ and !~ */ + tDOT2 tDOT3 /* .. and ... */ + tAREF tASET /* [] and []= */ + tLSHFT tRSHFT /* << and >> */ + tCOLON2 /* :: */ + tCOLON3 /* :: at EXPR_BEG */ + tASSOC /* => */ + tLPAREN /* ( */ + tLPAREN_ARG /* ( */ + tRPAREN /* ) */ + tLBRACK /* [ */ + tLBRACE /* { */ + tLBRACE_ARG /* { */ + tSTAR /* * */ + tAMPER /* & */ + tLAMBDA /* -> */ + tSYMBEG tSTRING_BEG tXSTRING_BEG tREGEXP_BEG tWORDS_BEG tQWORDS_BEG + tSTRING_DBEG tSTRING_DVAR tSTRING_END tLAMBEG + /*%%%*/ + /*% +%token '{' '}' '[' ']' '(' ')' '.' +%*/ + /* * precedence table */ @@ -1217,7 +1236,7 @@ /*%%%*/ $$ = NEW_CALL($1, $3, $4); /*% - $$ = dispatch3(call, $1, ripper_id2sym('.'), $3); + $$ = dispatch3(call, $1, $2, $3); $$ = method_arg($$, $4); %*/ } @@ -1226,7 +1245,7 @@ /*%%%*/ $$ = NEW_CALL($1, $3, $4); /*% - $$ = dispatch3(call, $1, ripper_intern("::"), $3); + $$ = dispatch3(call, $1, $2, $3); $$ = method_arg($$, $4); %*/ } @@ -2533,7 +2552,7 @@ /*%%%*/ $$ = $2; /*% - $$ = dispatch1(paren, $2); + $$ = dispatch3(paren, $1, $2, $4); %*/ } | tLPAREN compstmt ')' @@ -2541,7 +2560,7 @@ /*%%%*/ $$ = $2; /*% - $$ = dispatch1(paren, $2); + $$ = dispatch3(paren, $1, $2, $3); %*/ } | primary_value tCOLON2 tCONSTANT @@ -2570,7 +2589,7 @@ $$ = $2; } /*% - $$ = dispatch1(array, escape_Qundef($2)); + $$ = dispatch3(array, $1, escape_Qundef($2), $3); %*/ } | tLBRACE assoc_list '}' @@ -2578,7 +2597,7 @@ /*%%%*/ $$ = NEW_HASH($2); /*% - $$ = dispatch1(hash, escape_Qundef($2)); + $$ = dispatch3(hash, $1, escape_Qundef($2), $3); %*/ } | keyword_return @@ -2594,7 +2613,7 @@ /*%%%*/ $$ = new_yield($3); /*% - $$ = dispatch1(yield, dispatch1(paren, $3)); + $$ = dispatch1(yield, dispatch3(paren, $2, $3, $4)); %*/ } | keyword_yield '(' rparen @@ -2602,7 +2621,7 @@ /*%%%*/ $$ = NEW_YIELD(0, Qfalse); /*% - $$ = dispatch1(yield, dispatch1(paren, arg_new())); + $$ = dispatch1(yield, dispatch3(paren, $2, arg_new(), $3)); %*/ } | keyword_yield @@ -2870,7 +2889,7 @@ in_def--; cur_mid = $3; /*% - $$ = dispatch3(def, $2, $4, $5); + $$ = dispatch5(def, $1, $2, $4, $5, $6); in_def--; cur_mid = $3; %*/ @@ -2896,7 +2915,7 @@ local_pop(); in_single--; /*% - $$ = dispatch5(defs, $2, $3, $5, $7, $8); + $$ = dispatch7(defs, $1, $2, $3, $5, $7, $8, $9); in_single--; %*/ } @@ -3269,7 +3288,7 @@ /*%%%*/ $$ = NEW_LAMBDA($2); /*% - $$ = dispatch1(paren, $2); + $$ = dispatch3(paren, $1, dispatch2(f_larglist, $2, $3), $4); %*/ } | f_args opt_bv_decl @@ -3329,7 +3348,7 @@ /*%%%*/ $$ = NEW_CALL($1, $3, $4); /*% - $$ = dispatch3(call, $1, ripper_id2sym('.'), $3); + $$ = dispatch3(call, $1, $2, $3); $$ = method_optarg($$, $4); %*/ } @@ -3338,7 +3357,7 @@ /*%%%*/ $$ = NEW_CALL($1, $3, $4); /*% - $$ = dispatch3(call, $1, ripper_intern("::"), $3); + $$ = dispatch3(call, $1, $2, $3); $$ = method_optarg($$, $4); %*/ } @@ -3359,7 +3378,7 @@ $$ = NEW_CALL($1, $3, $4); fixpos($$, $1); /*% - $$ = dispatch3(call, $1, ripper_id2sym('.'), $3); + $$ = dispatch3(call, $1, $2, $3); $$ = method_optarg($$, $4); %*/ } @@ -3369,7 +3388,7 @@ $$ = NEW_CALL($1, $3, $4); fixpos($$, $1); /*% - $$ = dispatch3(call, $1, ripper_id2sym('.'), $3); + $$ = dispatch3(call, $1, $2, $3); $$ = method_optarg($$, $4); %*/ } @@ -3378,7 +3397,7 @@ /*%%%*/ $$ = NEW_CALL($1, $3, 0); /*% - $$ = dispatch3(call, $1, ripper_intern("::"), $3); + $$ = dispatch3(call, $1, $2, $3); %*/ } | primary_value '.' paren_args @@ -3387,8 +3406,7 @@ $$ = NEW_CALL($1, rb_intern("call"), $3); fixpos($$, $1); /*% - $$ = dispatch3(call, dispatch1(paren, $1), - ripper_id2sym('.'), rb_intern("call")); + $$ = dispatch3(call, $1, $2, ripper_intern("call")); $$ = method_optarg($$, $3); %*/ } @@ -3398,8 +3416,7 @@ $$ = NEW_CALL($1, rb_intern("call"), $3); fixpos($$, $1); /*% - $$ = dispatch3(call, dispatch1(paren, $1), - ripper_id2sym('.'), rb_intern("call")); + $$ = dispatch3(call, $1, $2, rb_intern("call")); $$ = method_optarg($$, $3); %*/ } @@ -3587,7 +3604,7 @@ /*%%%*/ $$ = $2; /*% - $$ = dispatch1(string_literal, $2); + $$ = dispatch3(string_literal, $1, $2, $3); %*/ } ; @@ -3614,7 +3631,7 @@ } $$ = node; /*% - $$ = dispatch1(xstring_literal, $2); + $$ = dispatch3(xstring_literal, $1, $2, $3); %*/ } ; @@ -3666,12 +3683,16 @@ /*%%%*/ $$ = NEW_ZARRAY(); /*% - $$ = dispatch0(words_new); + $$ = dispatch3(words, $1, dispatch0(words_new), $3); %*/ } | tWORDS_BEG word_list tSTRING_END { + /*%%%*/ $$ = $2; + /*% + $$ = dispatch3(words, $1, $2, $3); + %*/ } ; @@ -3716,12 +3737,16 @@ /*%%%*/ $$ = NEW_ZARRAY(); /*% - $$ = dispatch0(qwords_new); + $$ = dispatch3(qwords, $1, dispatch0(qwords_new), $3); %*/ } | tQWORDS_BEG qword_list tSTRING_END { + /*%%%*/ $$ = $2; + /*% + $$ = dispatch3(qwords, $1, $2, $3); + %*/ } ; @@ -3793,7 +3818,7 @@ $$ = NEW_EVSTR($3); /*% lex_strterm = $2; - $$ = dispatch1(string_dvar, $3); + $$ = dispatch2(string_dvar, $1, $3); %*/ } | tSTRING_DBEG @@ -3813,7 +3838,7 @@ if ($3) $3->flags &= ~NODE_NEWLINE; $$ = new_evstr($3); /*% - $$ = dispatch1(string_embexpr, $3); + $$ = dispatch3(string_embexpr, $1, $3, $4); %*/ } ; @@ -3852,7 +3877,7 @@ $$ = $2; /*% lex_state = EXPR_ENDARG; - $$ = dispatch1(symbol, $2); + $$ = dispatch2(symbol, $1, $2); %*/ } ; @@ -3889,7 +3914,7 @@ } /*% lex_state = EXPR_ENDARG; - $$ = dispatch1(dyna_symbol, $2); + $$ = dispatch3(dyna_symbol, $1, $2, $3); %*/ } ; @@ -3987,7 +4012,7 @@ lex_state = EXPR_BEG; command_start = Qtrue; /*% - $$ = dispatch1(paren, $2); + $$ = dispatch3(paren, $1, $2, $3); lex_state = EXPR_BEG; command_start = Qtrue; %*/ @@ -4351,7 +4376,7 @@ } $$ = $3; /*% - $$ = dispatch1(paren, $3); + $$ = dispatch3(paren, $1, $3, $4); %*/ } ; @@ -4439,9 +4464,21 @@ ; rparen : opt_nl ')' + { + /*%%%*/ + /*% + $$ = $2; + %*/ + } ; rbracket : opt_nl ']' + { + /*%%%*/ + /*% + $$ = $2; + %*/ + } ; trailer : /* none */ @@ -9527,6 +9564,64 @@ return rb_funcall(parser->value, mid, 5, a, b, c, d, e); } +static VALUE +ripper_dispatch6(struct parser_params *parser, ID mid, VALUE a, VALUE b, + VALUE c, VALUE d, VALUE e, VALUE f) +{ + validate(a); + validate(b); + validate(c); + validate(d); + validate(e); + validate(f); + return rb_funcall(parser->value, mid, 5, a, b, c, d, e, f); +} + +static VALUE +ripper_dispatch7(struct parser_params *parser, ID mid, VALUE a, VALUE b, + VALUE c, VALUE d, VALUE e, VALUE f, VALUE g) +{ + validate(a); + validate(b); + validate(c); + validate(d); + validate(e); + validate(f); + validate(g); + return rb_funcall(parser->value, mid, 5, a, b, c, d, e, f, g); +} + +static VALUE +ripper_dispatch8(struct parser_params *parser, ID mid, VALUE a, VALUE b, + VALUE c, VALUE d, VALUE e, VALUE f, VALUE g, VALUE h) +{ + validate(a); + validate(b); + validate(c); + validate(d); + validate(e); + validate(f); + validate(g); + validate(h); + return rb_funcall(parser->value, mid, 5, a, b, c, d, e, f, g, h); +} + +static VALUE +ripper_dispatch9(struct parser_params *parser, ID mid, VALUE a, VALUE b, + VALUE c, VALUE d, VALUE e, VALUE f, VALUE g, VALUE h, VALUE i) +{ + validate(a); + validate(b); + validate(c); + validate(d); + validate(e); + validate(f); + validate(g); + validate(h); + validate(i); + return rb_funcall(parser->value, mid, 5, a, b, c, d, e, f, g, h, i); +} + static const struct kw_assoc { ID id; const char *name; --Apple-Mail-2-1041676122 Content-Type: text/plain; charset=US-ASCII; format=flowed Content-Transfer-Encoding: 7bit --Apple-Mail-2-1041676122--