From: Pen Ttt <myocean135@...>
Date: 2010-08-27T23:22:24+09:00
Subject: Re: error handling with exceptions in multi-thread download

  require 'rubygems'
  require 'open-uri'
  require 'nokogiri'
  require 'mysql'
  require 'date'

class Data
  def  self.getdaystock()
  time0=Time.now
  i=0
  cdown=[]
  threads=[]
  cthreaddown=[]
  open("/home/pt/usa/stocklist","r"){|item|
      while  line=item.gets
        cdown<<line
      end}
  a=(fromdate.split("-")[1].to_i-1).to_s
  b=fromdate.split("-")[2].to_s
  c=fromdate.split("-")[0].to_s
  d=Time.now.mon.to_s
  e=Time.now.day.to_s
  f=Time.now.year.to_s
  cdown=cdown.map {|item|
  item='http://ichart.finance.yahoo.com/table.csv?s='+item.to_s+'&amp;a='+a+'&amp;b='+b+'&amp;c='+c+\
         '&amp;d='+d+'&amp;e='+e+'&amp;f='+f+'&amp;g=d&amp;ignore=.csv'}
  cdown.each.with_index {|item,idx|
    cthreaddown<<  item
    i=i+1
    if i==100 or  idx==cdown.size-1  then
      cthreaddown=cthreaddown.uniq
      for page_to_fetch in cthreaddown
      threads<<Thread.new(page_to_fetch) do |web|
         datafile="/home/pt/usa/stock/"+web.split("=")[1].to_s.gsub("&amp;a","")
         begin
           open(datafile,'a+'){|refile|
           open(web){|webfile|
              refile.puts  webfile.read
              puts  "#{web}  over"   }}
         rescue Timeout::Error => e
           puts    "#{web}  failed"
         rescue NameError => e
           puts    "#{web}  failed"
         rescue => e
           puts  "#{web}  wrong"
         end
       end
      end
      threads.each {|thr|
        begin
          thr.join
        rescue Timeout::Error => e
          puts    "#{web}  failed"
        rescue NameError => e
          puts    "#{web}  failed"
        rescue => e
          puts    "#{web}  failed"
        end    }
       i=0
       cthreaddown=[]
       threads = []
       sleep  0.01
    else
    end
    }
  end
 end

i have opened  100 threads. there are   6000 companies in 
/home/pt/usa/stocklist .

-- 
Posted via http://www.ruby-forum.com/.