From: Joey Marino <joey.da3rd@...>
Date: 2008-04-03T01:52:53+09:00
Subject: Re: GC help

------=_Part_2169_13192303.1207155165243
Content-Type: text/plain; charset=ISO-8859-1
Content-Transfer-Encoding: 7bit
Content-Disposition: inline

Ok, I was able to get it all into one class. The problem lies in this class:
class Picture

 def initialize(db,rets,rets_class)
   @db = db
   @rets = rets
   @rets_class = rets_class
   @attempts = 0
 end


 def getPic(key)
   begin
     get_object_request = GetObjectRequest.new(@rets_class, "Photo")
     get_object_request.add_all_objects(key)
     get_object_response = @rets.session.get_object(get_object_request)
     content_type_suffixes = { "image/jpeg" => "jpg"}
     makePicDir(key)
     get_object_response.each_object do |object_descriptor|
       object_key =  object_descriptor.object_key
       obj_id = object_descriptor.object_id
       content_type = object_descriptor.content_type
       description = object_descriptor.description
       #print "#{object_key} object \##{object_id}"
       #print ", description: #{description}" if !description.empty?
       #puts
       suffix = content_type_suffixes[content_type]
       pic = object_descriptor.data_as_string
       savePic(key,obj_id.to_s,suffix,description,pic)
     end
     get_object_response = nil
  rescue => e
    puts "Error retrieving pictures for #{key}: " + e
    if @attempts <= 5
      @attempts += 1
      puts "retrying"
      retry
    else
      puts "failed"
      @attempts = 0
    end
  end
  @attempts = 0
 end


 def getThumb(key)
   begin
     get_object_request = GetObjectRequest.new(@rets_class, "Thumbnail")
     get_object_request.add_all_objects(key)
     get_object_response = @rets.session.get_object(get_object_request)
     content_type_suffixes = { "image/jpeg" => "jpg"}
     get_object_response.each_object do |object_descriptor|
       object_key =  object_descriptor.object_key
       obj_id = object_descriptor.object_id
       content_type = object_descriptor.content_type
       description = object_descriptor.description
       #print "#{object_key} object \##{object_id}"
       #print ", description: #{description}" if !description.empty?
       #puts
       suffix = content_type_suffixes[content_type]
       pic = object_descriptor.data_as_string
       savePic(key,obj_id.to_s,suffix,description,pic,true)
     end
     get_object_response = nil
  rescue => e
    puts "Error retrieving thumbs for #{key}: " + e
    if @attempts <= 5
      @attempts += 1
      puts "retrying"
      retry
    else
      puts "failed"
      @attempts = 0
    end
  end
  @attempts = 0
 end


 def makePicDir(key)
   FileUtils.mkpath("#{$pic_dir}#{key}/thumb")
 end




 def savePic(key,id,suffix,desc,pic,thumb_bool=false)
   if thumb_bool
     file_name = $pic_dir + key + "/thumb/" + id + "." + suffix
     location = "/" + key + "/thumb/" + id + "." + suffix
   else
     file_name = $pic_dir + key + "/" + id + "." + suffix
     location = "/" + key + "/" + id + "." + suffix
   end
   self.savePicFile(file_name,pic)
   size = File.size(file_name)
   if thumb_bool
     self.insertThumbDB(key,id,location)
   else
     self.insertPicDB(key,id,desc,size,location)
   end
 end




 def savePicFile(file_name,pic)
   f = File.open(file_name, "wb")
   f << pic
   f.close
 end




 def insertPicDB(key,id,desc,size,location)
   description = @db.database.escape_string(desc)
   if
@db.DBinsert("PICS","pkey,id,description,size,location","#{key},#{id},'#{description}','#{size}','#{location}'")
   # puts "#{key} #{id} pic added"
     print ":"
   end
 end


 def insertThumbDB(key,id,location)
   if @db.DBupdate("PICS","thumb = '#{location}'"," pkey = #{key} and id =
#{id}")
    # puts "#{key} #{id} thumb added"
      print "."
   end
 end



 def deletePic(key)
   self.deletePicDir(key)
   self.deletePicDB(key)
 end


 def deletePicDir(key)
   if File.exists?("#{$pic_dir}#{key}")
     FileUtils.remove_dir("#{$pic_dir}#{key}")
   end
 end



 def deletePicDB(key)
  if @db.DBdelete("PICS","pkey = #{key}")
    print "-"
#    puts "#{key} pics deleted from db"
  end
 end

end #end class


On Wed, Apr 2, 2008 at 6:02 AM, Michal Suchanek <hramrach@centrum.cz> wrote:

> On 31/03/2008, Tim Hunter <TimHunter@nc.rr.com> wrote:
> > Joey Marino wrote:
> >
> > > I found out it is not the 3rd party code. I am downloading images
> > (~320,000
> > > ct.) as strings, and saving each to a file. The file handlers are
> getting
> > > closed. Using the Dike gem, I think I determined the bulk of the
> leaked
> > > memory is in these string variables. But what I don't understand is
> that
> > the
> > > variables containing the strings are local to a method of an object
> and
> > > should be unreferenced when that method is done, right? I've invoked
> GC at
> > > the end of the method and after each iteration that the method gets
> > called.
> > > I've even set the strings to nil after they've been saved to a file
> and
> > > before gc is called. The app is using 2G of ram and 4G of swap before
> it
> > > runs of out memory and crashes about 1/3rd of the way through. I'm
> really
> > > starting to doubt Ruby's ability to do memory intensive work. Any
> ideas?
> > >
> >
> >  Many people have used Ruby for long-running tasks that use a lot of
> memory.
> > If Ruby was not collecting unused strings, don't you think somebody
> would
> > have noticed it by now?
>
> I have noticed :->
>
> Michal
>
>


-- 
Joey Marino

------=_Part_2169_13192303.1207155165243--