From: William James <w_a_x_man@...>
Date: 2005-12-05T22:22:33+09:00
Subject: Re: Record-separator is a regular expression

Third version.  And here's an example of using it to remove
all html tags from a file:

File.open("data1.htm"){|handle|
  reader = RecSep.new( handle, /<.*?>/m )
  reader.each {|x| print x }
}

-----------------------------------------------------------

=begin

Unlike Gawk and Mawk, Ruby won't accept a regular expression as a
record-separator. Let's fix that. The substring matched by the
record-separator is automatically removed from the record, but it
can be obtained by RecSep#terminator.

Typical usage:

File.open("stuff.txt"){|handle|
  reader = RecSep.new( handle, /^\d+\.\n/ )
  reader.each {|x| p x }
}

Sometimes it may be necessary to keep the regular expression from
matching less than it should by increasing the look-ahead distance
(measured in characters):

File.open("stuff.txt"){|handle|
  reader = RecSep.new( handle, /(^.*\n)\1+/m, 4096 )
  reader.each {|x| p x }
}

=end

class RecSep

  def initialize( file_handle, record_separator,
      minimal_look_ahead = 1024 )
    @handle = file_handle
    @rec_sep = record_separator
    @min_look_ahead = minimal_look_ahead
    @buffer = ""
    @terminator = nil
    @count = 0
  end

  attr_reader :terminator, :count, :buffer

  def get_rec
    ##  Make sure the buffer has a reasonable amount of material.
    if @buffer.size < (3 * @min_look_ahead / 2)  &&  !@handle.eof?
      @buffer << @handle.read( 2 * @min_look_ahead - @buffer.size)
    end
    ##  To cope with all kinds of greedy regular expressions,
    ##  we read until there are at least @min_look_ahead bytes
    ##  left over in the buffer after the match.
    loop  do
      @rec_sep.match( @buffer )
      break  if  $~  &&  $~.post_match.size >= @min_look_ahead
      s = @handle.read( @min_look_ahead )
      break  if not s
      @buffer << s
    end

    if $~
      @buffer = $~.post_match
      @terminator = $~.to_s
      @count += 1
      $~.pre_match
    else
      @terminator = nil
      return nil  if "" == @buffer
      @count += 1
      s, @buffer = @buffer, ""
      s
    end
  end

  def each
    while s = self.get_rec
      yield s
    end
  end

end