From: Robert Klemme <bob.news@...>
Date: 2005-09-23T18:31:41+09:00
Subject: Re: Optimizing a single slow method

------=_NextPart_000_0079_01C5C032.02E72E10
Content-Type: text/plain;
	format=flowed;
	charset="iso-8859-1";
	reply-type=response
Content-Transfer-Encoding: 7bit


Glenn,

here's my first shot.  It doesn't seem very fast but then again there are 
still some uncertainties:

 - How many contracts are typically in a reference day's set?

 - How many contracts are there in total?

 - How many percent of the reference contracts are present in an average 
file?

 - How do dates relate to files? (I assumed a file per day plus I used 
synthetic dates; see the generator script)

For 50 files with 212998930 bytes this took 476.511s total on my machine 
(2.34s/MB).  Maybe you just throw it at your data set and see how it works 
out.

Kind regards

    robert

------=_NextPart_000_0079_01C5C032.02E72E10
Content-Type: application/octet-stream;
	name="generate_ticks.rb"
Content-Transfer-Encoding: quoted-printable
Content-Disposition: attachment;
	filename="generate_ticks.rb"

#!/usr/bin/ruby=0A=
=0A=
# a single line:=0A=
# A62005U,050909,0.7726,0.7758,0.7703,0.7737,12366,0=0A=
=0A=
def gen_id=0A=
  sprintf( "%s%05d%s", (?A+ rand(?Z - ?A + 1)).chr, rand(100_000),(?A+ =
rand(?Z - ?A + 1)).chr )=0A=
end=0A=
=0A=
def gen_stock=0A=
  rand(1000) / 100.0=0A=
end=0A=
=0A=
contract_ids =3D []=0A=
25_000.times { contract_ids << gen_id }=0A=
=0A=
File.open("ref.csv", "w") do |io|=0A=
  100_000.times do=0A=
    io.print contract_ids[rand(contract_ids.size)],=0A=
      ",050909,",=0A=
      gen_stock, ",",=0A=
      gen_stock, ",",=0A=
      gen_stock, ",",=0A=
      gen_stock, ",12366,0\n"=0A=
  end=0A=
end=0A=
=0A=
50_000.times { contract_ids << gen_id }=0A=
=0A=
date =3D "050809"=0A=
=0A=
50.times do |i|=0A=
  File.open("tick-#{"%02d" % i}.csv", "w") do |io|=0A=
    100_000.times do=0A=
      io.print contract_ids[rand(contract_ids.size)],=0A=
        ",", date, ",",=0A=
        gen_stock, ",",=0A=
        gen_stock, ",",=0A=
        gen_stock, ",",=0A=
        gen_stock, ",12366,0\n"=0A=
    end=0A=
  end=0A=
=0A=
  date =3D date.succ=0A=
end=0A=

------=_NextPart_000_0079_01C5C032.02E72E10
Content-Type: application/octet-stream;
	name="parse_contracts.rb"
Content-Transfer-Encoding: quoted-printable
Content-Disposition: attachment;
	filename="parse_contracts.rb"

#!/usr/bin/ruby=0A=
=0A=
=0A=
# a single line:=0A=
# A62005U,050909,0.7726,0.7758,0.7703,0.7737,12366,0=0A=
=0A=
class IllegalStateException < StandardError=0A=
end=0A=
=0A=
# -------------------------------=0A=
=0A=
class Contract=0A=
  Tick =3D Struct.new :date, :open, :high, :low, :close=0A=
=0A=
  class Tick=0A=
    include Comparable=0A=
=0A=
    def <=3D> (tick) self.date <=3D> tick.date end=0A=
  end=0A=
=0A=
  attr_reader :contract_id=0A=
=0A=
  def initialize(contract_id)=0A=
    @contract_id =3D contract_id=0A=
    @ticks =3D []=0A=
  end=0A=
=0A=
  def add_tick(date, open, high, low, close)=0A=
    @ticks << Tick.new( date, open, high, low, close )=0A=
  end=0A=
=0A=
  def sort_ticks() @ticks.sort! end=0A=
end=0A=
=0A=
# -------------------------------=0A=
=0A=
class ContractSet=0A=
  def parse_reference(io)=0A=
    @contracts =3D {}=0A=
=0A=
    io.each_line do |line|=0A=
      if /^(\w\d+\w),/ =3D~ line=0A=
        cid =3D $1.freeze=0A=
        @contracts[cid] ||=3D Contract.new cid=0A=
      end=0A=
    end=0A=
  end=0A=
=0A=
  def parse_ticks(io)=0A=
    raise IllegalStateException, "need to read reference first" unless =
@contracts=0A=
=0A=
    io.each_line do |line|=0A=
      if %r{^=0A=
             (\w\d+\w),         # contractid=0A=
             (\d{6}),           # date=0A=
             (\d+(?:\.\d+)?),   # open=0A=
             (\d+(?:\.\d+)?),   # high=0A=
             (\d+(?:\.\d+)?),   # low=0A=
             (\d+(?:\.\d+)?),   # close=0A=
           }x =3D~ line=0A=
        cid =3D $1.freeze=0A=
=0A=
        contract =3D @contracts[cid] and=0A=
          contract.add_tick $2, $3.to_f, $4.to_f, $5.to_f, $6.to_f=0A=
      end=0A=
    end=0A=
  end=0A=
=0A=
  def post_process=0A=
    @contracts.each_value do |contract|=0A=
      contract.sort_ticks=0A=
    end=0A=
  end=0A=
end=0A=
=0A=
class Ticker=0A=
  def initialize()=0A=
    @t =3D Time.now.to_f=0A=
    @count =3D 0.0=0A=
  end=0A=
=0A=
  def tick(msg)=0A=
    diff =3D Time.now.to_f - @t=0A=
    @count +=3D diff=0A=
    print diff, msg=0A=
    @t =3D Time.now.to_f=0A=
  end=0A=
=0A=
  def total=0A=
    print @count, "total"=0A=
  end=0A=
=0A=
  def print(diff, msg)=0A=
    printf "%8.3fs %s\n", diff, msg=0A=
  end=0A=
end=0A=
=0A=
# =
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=0A=
# MAIN=0A=
# =
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=0A=
=0A=
t =3D Ticker.new=0A=
cs =3D ContractSet.new=0A=
=0A=
File.open(ARGV.shift, IO::RDONLY) {|io| cs.parse_reference io}=0A=
t.tick "read ref"=0A=
=0A=
ARGV.each do |f|=0A=
  File.open(f, IO::RDONLY) {|io| cs.parse_ticks io}=0A=
  t.tick "read file #{f}"=0A=
end=0A=
=0A=
cs.post_process=0A=
=0A=
t.tick "finished post processing"=0A=
t.total=0A=

------=_NextPart_000_0079_01C5C032.02E72E10--