From: Stefan Lang <langstefan@...>
Date: 2005-10-30T23:55:42+09:00
Subject: Re: Faster CSV parsing

--Boundary-00=_98NZDJwsHvMffL+
Content-Type: Multipart/Mixed;
  boundary="Boundary-00=_98NZDJwsHvMffL+";
  charset="iso-8859-1"

On Sunday 30 October 2005 14:52, gabriele renzi wrote:
> William James ha scritto:
[...]
> > For my test file of 1,964,211 bytes, it's about 6.4 times as
> > fast.
>
> what things is this missing wrt standard csv.rb?
>
> Also, why you did choose to make all of the stuff (methods,
> variables) at class level instead of instance ?

A more OO (and equally fast) version:

## Read, parse, and create csv records.

# The program conforms to the csv specification at this site:
# http://www.creativyst.com/Doc/Articles/CSV/CSV01.htm
# The only extra is that you can change the field-separator.
# For a field-separator other than a comma, for example
# a semicolon:
#   csv.fs=";"  # csv is a FastCsv instance
#
# After a record has been read and parsed,
# csv.string contains the record in raw string format.
#

class FastCsv

    def self.foreach(filename)
        csv = self.new
        open filename do |file|
            while record = csv.get_rec(file)
                yield record
            end
        end
    end

    def initialize(fs = ",")
        self.fs = fs
        @string = nil
    end

    def fs=(s)
        raise "fs must be a single character." if s.size != 1
        @fs = s.dup
        make_regexp
    end

    def fs
        @fs.dup
    end

    def parse(s)
        ary = (s + @fs).scan(@regexp)
        raise "Bad csv record:\n#{s}\n" if $' != ""
        unescape(ary.flatten.compact)
    end

    def get_rec(file)
        return nil if file.eof?
        @string = ""
        begin
            if @string.size > 0
                raise "Bad record:\n#@string\n" if @string !~ 
@reading_regexp
                raise "Premature end of csv file." if file.eof?
            end
            @string += file.gets
        end until @string.count('"') % 2 == 0
        @string.chomp!
        parse(@string)
    end

    def string
        @string
    end

    def to_csv(array)
        s = ''
        array.map { |item|
            str = item.to_s
            # Quote the string if it contains the field-separator or
            # a " or a newline or a carriage-return, or if it has 
leading or
            # trailing whitespace.
            if str.index(@fs) or /^\s|["\r\n]|\s$/.match(str)
                str = '"' + str.gsub( /"/, '""' ) + '"'
            end
            str
        }.join(@fs)
    end

    private

    def unescape(array)
        array.map { |x| x.gsub(/""/, '"') }
    end

    # Set regexp for parse.
    # @fs is the field-separator, which must be
    # a single character.
    def make_regexp
        fs = @fs
        if "^" == fs
            fs = "\\^"
        end

        @regexp =
            ## Assumes embedded quotes are escaped as "".
            %r{
           \G   ## Anchor at end of previous match.
           [ \t]*   ## Leading spaces or tabs are discarded.
           (?:
               ##  For a field in quotes.
               " (  [^"]*  (?:  "" [^"]*  )*  ) "  |
               ##  For a field not in quotes.
                 (  [^"\n#{fs}]*?  )
           )
           [ \t]*
           [#{fs}]
        }mx

        ## When get_rec finds after reading a line that the record 
isn't
        ## complete, this regexp will be used to decide whether to 
read
        ## another line or to raise an exception.
        @reading_regexp =
            %r{
          \A      # Anchor at beginning of string.
          (?:
             [ \t]*
             (?:
                 "  [^"]*  (?:  "" [^"]*  )*  "  |
                    [^"\n#{fs}]*?
             )
             [ \t]*
             [#{fs}]
          )*
          [ \t]*
          "  [^"]*  (?:  "" [^"]*  )*
          \Z      # Anchor at end of string.
        }mx
    end # make_regexp

end # class FastCsv

if $0 == __FILE__
    FastCsv.foreach("example.csv") { |record| p record }
end
--Boundary-00=_98NZDJwsHvMffL+--