Re: net/ftp is cpu bound on fast internet connections

Hi,

> I was trying to run multiple ftp.putbinaryfile jobs and noticed that if 
> I run more than say 2 threads on my PIII 933, My CPU usage hits 100% and 
> my IO stays about the same, even though I am nowhere close to my 
> theorectical limit.  Here's the original code.
> 
> 1.upto(5) {|server_suffix|
>    puts "starting send to DL360#{server_suffix}"
>    threads[server_suffix] = Thread.new {
> 	ftp[server_suffix].putbinaryfile '100recs.tar.gz',
> 					 '100recs.tar.gz'
> 
>          telnet[server_suffix].cmd 'tar -xzf 100recs.tar.gz'
>    }
> }
> threads[1..5].each {|thread| thread.join}
> 
> My connection should actually be able to handle about 20 of these 
> threads at a time.  What I noticed was that of course ftp.rb's routines 
> are block oriented and pass everything through yield to a block for 
> processing.  I was wondering if it would be good for me or someone to 
> add non block-oriented functions that perhaps even memory map and or 
> buffer files where appropriate to save cpu time.

Can you try this patch?

Shugo

Index: lib/net/ftp.rb
===================================================================
RCS file: /src/ruby/lib/net/ftp.rb,v
retrieving revision 1.10
diff -u -r1.10 ftp.rb
--- lib/net/ftp.rb	2002/06/11 04:20:11	1.10
+++ lib/net/ftp.rb	2002/06/11 08:35:51
@@ -261,26 +261,21 @@
       @welcome = resp
     end
     
-    def retrbinary(cmd, blocksize, rest_offset = nil, callback = Proc.new)
+    def retrbinary(cmd, blocksize, rest_offset = nil)
       synchronize do
 	voidcmd("TYPE I")
 	conn = transfercmd(cmd, rest_offset)
 	loop do
 	  data = conn.read(blocksize)
 	  break if data == nil
-	  callback.call(data)
+	  yield(data)
 	end
 	conn.close
 	voidresp
       end
     end
     
-    def retrlines(cmd, callback = nil)
-      if block_given?
-	callback = Proc.new
-      elsif not callback.is_a?(Proc)
-	callback = Proc.new {|line| print line, "\n"}
-      end
+    def retrlines(cmd)
       synchronize do
 	voidcmd("TYPE A")
 	conn = transfercmd(cmd)
@@ -292,18 +287,14 @@
 	  elsif line[-1] == ?\n
 	    line = line[0 .. -2]
 	  end
-	  callback.call(line)
+	  yield(line)
 	end
 	conn.close
 	voidresp
       end
     end
     
-    def storbinary(cmd, file, blocksize, rest_offset = nil, callback = nil)
-      if block_given?
-	callback = Proc.new
-      end
-      use_callback = callback.is_a?(Proc)
+    def storbinary(cmd, file, blocksize, rest_offset = nil, &block)
       synchronize do
 	voidcmd("TYPE I")
 	conn = transfercmd(cmd, rest_offset)
@@ -311,18 +302,14 @@
 	  buf = file.read(blocksize)
 	  break if buf == nil
 	  conn.write(buf)
-	  callback.call(buf) if use_callback
+	  yield(buf) if block
 	end
 	conn.close
 	voidresp
       end
     end
     
-    def storlines(cmd, file, callback = nil)
-      if block_given?
-	callback = Proc.new
-      end
-      use_callback = callback.is_a?(Proc)
+    def storlines(cmd, file, &block)
       synchronize do
 	voidcmd("TYPE A")
 	conn = transfercmd(cmd)
@@ -333,7 +320,7 @@
 	    buf = buf.chomp + CRLF
 	  end
 	  conn.write(buf)
-	  callback.call(buf) if use_callback
+	  yield(buf) if block
 	end
 	conn.close
 	voidresp
@@ -341,11 +328,7 @@
     end
     
     def getbinaryfile(remotefile, localfile,
-		      blocksize = DEFAULT_BLOCKSIZE, callback = nil)
-      if block_given?
-	callback = Proc.new
-      end
-      use_callback = callback.is_a?(Proc)
+		      blocksize = DEFAULT_BLOCKSIZE, &block)
       if @resume
 	rest_offset = File.size?(localfile)
 	f = open(localfile, "a")
@@ -357,24 +340,20 @@
 	f.binmode
 	retrbinary("RETR " + remotefile, blocksize, rest_offset) do |data|
 	  f.write(data)
-	  callback.call(data) if use_callback
+	  yield(data) if block
 	end
       ensure
 	f.close
       end
     end
     
-    def gettextfile(remotefile, localfile, callback = nil)
-      if block_given?
-	callback = Proc.new
-      end
-      use_callback = callback.is_a?(Proc)
+    def gettextfile(remotefile, localfile, &block)
       f = open(localfile, "w")
       begin
 	retrlines("RETR " + remotefile) do |line|
 	  line = line + @return_code
 	  f.write(line)
-	  callback.call(line) if use_callback
+	  yield(line) if block
 	end
       ensure
 	f.close
@@ -382,11 +361,7 @@
     end
     
     def putbinaryfile(localfile, remotefile,
-		      blocksize = DEFAULT_BLOCKSIZE, callback = nil)
-      if block_given?
-	callback = Proc.new
-      end
-      use_callback = callback.is_a?(Proc)
+		      blocksize = DEFAULT_BLOCKSIZE, &block)
       if @resume
 	rest_offset = size(remotefile)
       else
@@ -395,24 +370,16 @@
       f = open(localfile)
       begin
 	f.binmode
-	storbinary("STOR " + remotefile, f, blocksize, rest_offset) do |data|
-	  callback.call(data) if use_callback
-	end
+	storbinary("STOR " + remotefile, f, blocksize, rest_offset, &block)
       ensure
 	f.close
       end
     end
     
-    def puttextfile(localfile, remotefile, callback = nil)
-      if block_given?
-	callback = Proc.new
-      end
-      use_callback = callback.is_a?(Proc)
+    def puttextfile(localfile, remotefile, &block)
       f = open(localfile)
       begin
-	storlines("STOR " + remotefile, f) do |line|
-	  callback.call(line) if use_callback
-	end
+	storlines("STOR " + remotefile, f, &block)
       ensure
 	f.close
       end