Re: Regexp::Parser ported to Ruby...?

In message <[e-mail]>,
"Jeff 'japhy' Pinyan" <[e-mail]> writes:
>                         Some people would like the ability to assign
> DIRECTLY to some variable, like:
> 
>   "japhy" =~ /(?<$front>..)(?<$back>..)$/;
> 
> Well, this syntax isn't possible in Perl (yet).  

That's pretty funky, so I did some fiddling and now Ruby lets you do
   
    "japhy" =~ /(?[front]..)(?[back]..)$/
	puts "front=#{$front} back=#{$back}"

Caveat Emptor: I knocked this up in short space of time and
it's an inelegant hack, but it mostly works.  It only defines global
variables because the first way I tried (rb_eval("variable=%q{match}")
broke on things like "2{=" and it was easier to switch to
rb_define_variable than to handle borderline cases like that.

It doesn't check that the variable you're passing is legal, valid,
sensible or even vaguely presentable.  It trusts you not to hurt it.

I'm not sure the memory management is sensible (I use strdup at one point,
but I guess I could easily change that to use rb_str_new() and store the
Ruby string instead of the C string.  The only documentation I had to work
from was README.EXT in the source tarball.

Variables are only wiped when the whole regexp fails, not individually,
which is suboptimal but better than nothing for the moment.

The patch is against the 1.6.6 sources, and should compile cleanly.
Let me know if I've missed anything glaringly obvious or if there's
anything else I need on the TODO list.

TODO: 
* variable deletions when part of a regexp fails
  "japhy" =~ /(?[a]ap)(?[b]z?)/ # should b end up as nil or ""?
* memory management clean ups
* rewrite the whole thing more elegantly
* backpatch the functionality into GNU regex code for general use?
  (so re_match calls hooks for (?[string]regexp) patterns)

diff -burP ruby-1.6.6o/re.c ruby-1.6.6/re.c
--- ruby-1.6.6o/re.c	Tue Dec 25 14:24:35 2001
+++ ruby-1.6.6/re.c	Thu Feb  7 22:48:23 2002
@@ -623,6 +623,15 @@
     }
 
     if (result < 0) {
+      struct re_pattern_buffer *pb = RREGEXP(re)->ptr;
+      int i;
+
+      /* reset all the variables */
+      for (i=1; i<=pb->regvars; i++) {
+        if (pb->regvar[i]) {
+          rb_define_variable(pb->regvar[i], NULL); /* seems to work */
+        }
+      }
	rb_backref_set(Qnil);
	return result;
     }
@@ -647,6 +656,20 @@
     RMATCH(match)->str = rb_str_new4(str);
     rb_backref_set(match);
 
+    if(RMATCH(match)->regs->num_regs > 0 && RMATCH(match)->regs->regvars > 0) {
+      int i;
+      struct re_registers *r = RMATCH(match)->regs;
+      for (i=1; i<r->num_regs; i++) {
+        char e[1024];
+        VALUE *s = ALLOC_N(VALUE, 1);
+        *s = rb_reg_nth_match(i, match);
+        if (r->var[i] && *s) {
+          rb_define_variable(r->var[i], s);
+        }
+        r->var[i] = NULL;
+      }  
+    }
+    
     OBJ_INFECT(match, re);
     OBJ_INFECT(match, str);
     return result;
diff -burP ruby-1.6.6o/regex.c ruby-1.6.6/regex.c
--- ruby-1.6.6o/regex.c	Wed Dec  5 14:36:41 2001
+++ ruby-1.6.6/regex.c	Thu Feb  7 22:50:00 2002
@@ -1261,6 +1261,10 @@
 
   int options = bufp->options;
 
+  if (bufp->regvar) xfree(bufp->regvar);
+  bufp->regvar = NULL;
+  bufp->regvars = 0;
+
   bufp->fastmap_accurate = 0;
   bufp->must = 0;
   bufp->must_skip = 0;
@@ -1777,6 +1781,25 @@
 	case '>':
 	  break;
 
+    /* this is horrible */
+    case '[': {
+      char var[1024]; /* hackeroo */
+      int j=0;
+      for (;;) {
+        PATFETCH(c);
+        if (c==']') break;
+        var[j++]=c;
+        var[j]='\0';
+      }
+      if (regnum > bufp->regvars) {
+          TREALLOC(bufp->regvar, regnum+1, unsigned char *);
+      }
+      bufp->regvar[regnum] = strdup(var);
+      bufp->regvars = regnum;
+      c='('; /* pretend we're still parsing a match */
+      break; 
+    }
+
 	default:
 	  FREE_AND_RETURN(stackb, "undefined (?...) sequence");
 	}
@@ -2440,6 +2463,7 @@
   bufp->reg_info = TMALLOC(regnum, register_info_type);
   bufp->best_regstart = TMALLOC(regnum, unsigned char*);
   bufp->best_regend = TMALLOC(regnum, unsigned char*);
+  TREALLOC(bufp->regvar, regnum, unsigned char *);
   FREE_AND_RETURN(stackb, 0);
 
  invalid_pattern:
@@ -2476,6 +2500,7 @@
   xfree(bufp->best_regstart);
   xfree(bufp->best_regend);
   xfree(bufp->reg_info);
+  xfree(bufp->regvar);
   xfree(bufp);
 }
 
@@ -3495,15 +3520,18 @@
   if (regs->allocated == 0) {
     regs->beg = TMALLOC(num_regs, int);
     regs->end = TMALLOC(num_regs, int);
+    regs->var = TMALLOC(num_regs, char *);
     regs->allocated = num_regs;
   }
   else if (regs->allocated < num_regs) {
     TREALLOC(regs->beg, num_regs, int);
     TREALLOC(regs->end, num_regs, int);
+    TREALLOC(regs->var, num_regs, char *);
     regs->allocated = num_regs;
   }
   for (i=0; i<num_regs; i++) {
     regs->beg[i] = regs->end[i] = -1;
+    regs->var[i] = NULL;
   }
 }
 
@@ -3568,6 +3596,7 @@
 
   unsigned char **regstart = bufp->regstart;
   unsigned char **regend = bufp->regend;
+  unsigned char **regvar = bufp->regvar;
 
   /* If a group that's operated upon by a repetition operator fails to
      match anything, then the register for its start will need to be
@@ -3694,14 +3723,22 @@
       if (regs) {
 	regs->beg[0] = pos;
 	regs->end[0] = d - string;
+   regs->var[0] = NULL;
+   regs->regvars = bufp->regvars;
+
+   /* bleh */
+   TREALLOC(bufp->regvar, num_regs+1, unsigned char *);
+
 	for (mcnt = 1; mcnt < num_regs; mcnt++) {
 	  if (REG_UNSET(regend[mcnt])) {
 	    regs->beg[mcnt] = -1;
 	    regs->end[mcnt] = -1;
+       regs->var[mcnt] = NULL;
 	    continue;
 	  }
 	  regs->beg[mcnt] = regstart[mcnt] - string;
 	  regs->end[mcnt] = regend[mcnt] - string;
+     regs->var[mcnt] = bufp->regvar[mcnt];
 	}
       }
       FREE_AND_RETURN(stackb, (d - pos - string));
@@ -4405,18 +4442,22 @@
   if (regs1->allocated == 0) {
     regs1->beg = TMALLOC(regs2->num_regs, int);
     regs1->end = TMALLOC(regs2->num_regs, int);
+    regs1->var = TMALLOC(regs2->num_regs, char *);
     regs1->allocated = regs2->num_regs;
   }
   else if (regs1->allocated < regs2->num_regs) {
     TREALLOC(regs1->beg, regs2->num_regs, int);
     TREALLOC(regs1->end, regs2->num_regs, int);
+    TREALLOC(regs1->var, regs2->num_regs, char *);
     regs1->allocated = regs2->num_regs;
   }
   for (i=0; i<regs2->num_regs; i++) {
     regs1->beg[i] = regs2->beg[i];
     regs1->end[i] = regs2->end[i];
+    regs1->var[i] = regs2->var[i];
   }
   regs1->num_regs = regs2->num_regs;
+  regs1->regvars = regs2->regvars;
 }
 
 void
@@ -4426,6 +4467,7 @@
   if (regs->allocated == 0) return;
   if (regs->beg) xfree(regs->beg);
   if (regs->end) xfree(regs->end);
+  if (regs->var) xfree(regs->var);
 }
 
 /* Functions for multi-byte support.
diff -burP ruby-1.6.6o/regex.h ruby-1.6.6/regex.h
--- ruby-1.6.6o/regex.h	Fri Jul 13 06:48:25 2001
+++ ruby-1.6.6/regex.h	Thu Feb  7 18:06:00 2002
@@ -154,6 +154,10 @@
     register_info_type *reg_info;
     unsigned char **best_regstart;
     unsigned char **best_regend;
+
+    /* list of variable names to store register matches in */
+    unsigned char **regvar; 
+    int regvars;
   };
 
 typedef struct re_pattern_buffer regex_t;
@@ -172,8 +176,10 @@
   {
     int allocated;
     int num_regs;
+    int regvars;
     int *beg;
     int *end;
+    char **var;
   };
 
 /* Type for byte offsets within the string.  POSIX mandates this.  */
diff -burP ruby-1.6.6o/sample/test.rb ruby-1.6.6/sample/test.rb
--- ruby-1.6.6o/sample/test.rb	Wed Mar 21 08:04:12 2001
+++ ruby-1.6.6/sample/test.rb	Thu Feb  7 20:49:53 2002
@@ -674,6 +674,8 @@
 test_ok(/(\s+\d+){2}/ =~ " 1 2" && $& == " 1 2")
 test_ok(/(?:\s+\d+){2}/ =~ " 1 2" && $& == " 1 2")
 
+test_ok("moocowfish" =~ /(?[cow]c..)/ && $cow == "cow")
+
 $x = <<END;
 ABCD
 ABCD
require 'rubyunit'

class TestPermute < RUNIT::TestCase
  def test_simple
    testme=%q{moomopcowfishcowx{1=>2}y[1,2]z"string"y3}

    testme =~ /(c..)/      ;     assert($1 == "cow")
    testme =~ /(?[c]m..)/  ;     assert($c == "moo")
	                             assert($1 == "moo")
    testme =~ /(?[c]c.*?w)/;     assert($c == "cow")
	testme =~ /(?[c]x.*?)y/;     assert($c == "x{1=>2}")
	testme =~ /(?[c]y.*?)z/;     assert($c == "y[1,2]")
	testme =~ /(?[c]z.*)$/ ;     assert($c == %q{z"string"y3})
	testme =~ /(?[c]z.*)n/ ;     assert($c == %q{z"stri})
	testme =~ /(?[c]x.*)y/ ;     assert($c == %q{x{1=>2}y[1,2]z"string"})
	testme =~ /(?[c]x.*?)>/;     assert($c == "x{1=")
	testme =~ /y\[1(?[c].*?)z/;  assert($c == ",2]")

	testme =~ /(?[c]cow)(?[d].*cow)(.)/
		assert($c == "cow")
		assert($d == "fishcow")
		assert($3 == "x")

# failing regexps blank the variables
    testme =~ /(?[c]^m.p)/
        assert($c == nil)
        assert($d == "fishcow")

	"japhy" =~ /(?[front]..)(?[back]..)$/
		assert($front == "ap")
		assert($back  == "hy")
  end
end
-- 
rob partington % [e-mail] % http://lynx.browser.org/