From: Tanaka Akira <akr@...17n.org>
Date: 2002-02-21T13:31:32+09:00
Subject: [ruby-dev:16015] Re: making net/*.rb grok URI

In article <200202202350.g1KNoc731179@sharui.nakada.kanuma.tochigi.jp>,
  nobu.nakada@nifty.ne.jp writes:

> URI.open$B$G$O!#(B

$B$=$j$c$=$&$G$9$M!#$=$&$7$^$7$g$&!#(B

$B$G$b!"(BKernel#open $B$N=$@5$K$b<f$+$l$k$b$N$,$"$j$^$9!#(Bopen-uri.rb $B$H$+JL(B
$B$N%U%!%$%k$,$$$$$N$+$J!#(B

$B$H$$$&$o$1$GE,Ev$K:n$C$F$_$^$7$?!#(B
# $BFq$7$$$H$3$m$O<BAu$7$F$^$;$s$,!#(B

  URI.open("http://www.ruby-lang.org/en/index.html") {|f|
    f.each_line {|line| p line}
    p f.content_type
    p f.content_encoding
    p f.last_modified
  }

$B$H$$$&$h$&$K;H$($^$9!#(B

uri/open.rb $B$H$$$&$h$&$K?7$7$$%U%!%$%k$r:n$k$+!"$"$k$$$O(B
uri/generic.rb, uri/http.rb, uri/ftp.rb $B$"$?$j$KJ,$1$k$+!"$I$C$A$,$$$$(B
$B$+$O$h$/$o$+$j$^$;$s!#$"$H!"$D$$(B StringIO $B$r;H$C$F$7$^$C$?$N$G!"EvA3(B
StringIO $B$,K\BN$KF~$k$^$G$OK\BN$K$OF~$l$i$l$^$;$s!#(BStringIO $B$r;H$&$H$3(B
$B$m$r30$9$H$$$&$3$H$O$G$-$^$9$,!#(Brough $B$N(B uri $B$KF~$l$k$+$I$&$+$O(B... $B$d(B
$B$^$@$5$s$N0U8+$rJ9$-$?$$$H$3$m$G$9!#(B

$B>/$7;H$C$F;W$C$?$N$G$9$,!"(BStringIO $B$H(B IO $B$G6&DL$K;H$($k(B size/length $B$H(B
$B$$$&%a%=%C%I$,M_$7$$$G$9!#Cf?H$O$=$l$>$l(B self.string.size $B$H(B
self.stat.size $B$G$$$$$s$G$9$,!":#$O(B polymorphic $B$K;H$($k$b$N$,$"$j$^$;(B
$B$s!#(B

$B$"$H!"(BIO#truncate ($B$H$`$m$s(B StringIO#truncate)$B$,$J$$$3$H$K$b5$$,$D$-$^(B
$B$7$?!#(Bftruncate $B$O(B SUSv2 $B$K$b$"$j$^$9$+$i!"$"$C$F$b$$$$$s$8$c$J$$$+$H(B
$B;W$$$^$9!#$3$l$,$J$$$H%P%C%U%!$r@Z$j5M$a$i$l$^$;$s!#(B

require 'uri'
require 'stringio'
require 'tempfile'

module URI
  def URI.open(uri, *rest, &block)
    uri = URI.parse(uri) if String === uri
    uri.open(*rest, &block)
  end

  def URI.find_proxy(uri)
    name = uri.scheme + '_proxy'
    if proxy_uri = ENV[name] || ENV[name.upcase]
      proxy_uri = URI.parse(proxy_uri)
      name = 'no_proxy'
      if no_proxy = ENV[name] || ENV[name.upcase]
        no_proxy.scan(/([^:,]*)(?::(\d+))?/) {|host, port|
	  if /(\A|\.)#{Regexp.quote host}\z/i =~ proxy_uri.host &&
	     (!port || uri.port == port.to_i)
	    proxy_uri = nil
	    break
	  end
	}
      end
      proxy_uri
    else
      nil
    end
  end

  class HTTP
    def open(&block)
      Buf.open(lambda {|buf|
	require 'net/http'

	if proxy_uri = URI.find_proxy(self)
	  klass = Net::HTTP.Proxy(proxy_uri.host, proxy_uri.port)
	else
	  klass = Net::HTTP
	end

	resp = nil
	klass.start(self.host, self.port) {|http|
	  header = {}
	  resp, = http.get(self.request_uri, header, buf)
	}

	# If Redirection 3xx ...
	# If 401 Unauthorized ...

	unless resp.code == '200'
	  raise StandardError.new("HTTP fetch failed")
	end

	buf.stream.extend HTTPInfo
	buf.stream.init_httpinfo(resp)
      },
      &block)
    end
  end

  class FTP
    def open(&block)
      Buf.open(lambda {|buf|
	if proxy_uri = URI.find_proxy(uri)
	  require 'net/http'

	  resp = nil
	  Net::HTTP.start(proxy_uri.host, proxy_uri.port) {|http|
	    resp, = http.get(uri.to_s, nil, buf)
	  }

	  buf.stream.extend HTTPInfo
	  buf.stream.init_httpinfo(resp)
	else
	  require 'net/ftp'

	  # xxx: extract user/passwd from .netrc.
	  user = 'anonymous'
	  passwd = nil
	  if uri.userinfo
	    user, passwd = uri.userinfo.split(/:/)
	  end

	  ftp = Net::FTP.open(uri.host)
	  ftp.login(user, passwd)
	  ftp.getbinaryfile(uri.path, '/dev/null', Net::FTP::DEFAULT_BLOCKSIZE,
	    lambda {|data| buf << data})
	  ftp.close
	end
      },
      &block)
    end
  end

  module HTTPInfo
    def init_httpinfo(resp)
      @http_resp = resp
    end
    attr_reader :http_resp

    def content_type
      @http_resp['content-type']
    end

    def content_encoding
      @http_resp['content-encoding']
    end

    def last_modified
      if @http_resp.key? 'last-modified'
        require 'time'
        Time.httpdate(@http_resp['last-modified'])
      else
        nil
      end
    end
  end

  class Buf
    def Buf.open(init_contents=nil)
      buf = Buf.new
      if init_contents
	init_contents.call buf
	buf.stream.rewind
      end
      stream = buf.stream
      if block_given?
	begin
	  yield stream
	ensure
	  stream.close
	end
      else
	stream
      end
    end

    def initialize(basename='ruby-uri', size_limit=10240)
      @basename = basename
      @size_limit = size_limit
      @stream = StringIO.new('', 'w+')
      @size = 0
    end
    attr_reader :stream

    def <<(data)
      @size += data.length
      if @size_limit < @size && StringIO === @stream
	f = Tempfile.new(@basename)
	f << @stream.string
	@stream = f
      end
      @stream << data
    end
  end
end

module Kernel
  alias uri_original_open open
  def open(name, *rest, &block)
    if %r{\A(http|ftp)://} =~ name
      URI.open(name, *rest, &block)
    else
      uri_original_open(name, *rest, &block)
    end
  end
end
-- 
[$BEDCf(B $BE/(B][$B$?$J$+(B $B$"$-$i(B][Tanaka Akira]
$B!V$U$($m(B! $B$o$+$a$A$c$s:n@o$G$9$(C"=$B!W(B(Little Worker, $B7KM7@84](B)