alias open_uri_original_open open # :nodoc:
# makes possible to open various resources including URIs.
# If the first argument respond to `open' method,
# the method is called with the rest arguments.
# If the first argument is a string which begins with xxx://,
# it is parsed by URI.parse. If the parsed object respond to `open' method,
# the method is called with the rest arguments.
# Otherwise original open is called.
# Since open-uri.rb provides URI::HTTP#open, URI::HTTPS#open and
# Kernel[#.]open can accepts such URIs and strings which begins with
# http://, https:// and ftp://.
# In these case, the opened file object is extended by OpenURI::Meta.
def open(name, *rest, &block) # :doc:
if name.respond_to?(:open)
elsif name.respond_to?(:to_str) &&
%r{\A[A-Za-z][A-Za-z0-9+\-\.]*://} =~ name &&
(uri = URI.parse(name)).respond_to?(:open)
open_uri_original_open(name, *rest, &block)
# OpenURI is an easy-to-use wrapper for net/http, net/https and net/ftp.
# It is possible to open http/https/ftp URL as usual like opening a file:
# open("http://www.ruby-lang.org/") {|f|
# f.each_line {|line| p line}
# The opened file has several methods for meta information as follows since
# it is extended by OpenURI::Meta.
# open("http://www.ruby-lang.org/en") {|f|
# f.each_line {|line| p line}
# p f.base_uri # <URI::HTTP:0x40e6ef2 URL:http://www.ruby-lang.org/en/>
# p f.content_type # "text/html"
# p f.charset # "iso-8859-1"
# p f.content_encoding # []
# p f.last_modified # Thu Dec 05 02:45:02 UTC 2002
# Additional header fields can be specified by an optional hash argument.
# open("http://www.ruby-lang.org/en/",
# "User-Agent" => "Ruby/#{RUBY_VERSION}",
# "From" => "foo@bar.invalid",
# "Referer" => "http://www.ruby-lang.org/") {|f|
# The environment variables such as http_proxy, https_proxy and ftp_proxy
# are in effect by default. :proxy => nil disables proxy.
# open("http://www.ruby-lang.org/en/raa.html", :proxy => nil) {|f|
# URI objects can be opened in a similar way.
# uri = URI.parse("http://www.ruby-lang.org/en/")
# URI objects can be read directly. The returned string is also extended by
# Author:: Tanaka Akira <akr@m17n.org>
:content_length_proc => true,
:http_basic_authentication => true,
def OpenURI.check_options(options) # :nodoc:
unless Options.include? k
raise ArgumentError, "unrecognized option: #{k}"
def OpenURI.scan_open_optional_arguments(*rest) # :nodoc:
if !rest.empty? && (String === rest.first || Integer === rest.first)
if !rest.empty? && Integer === rest.first
def OpenURI.open_uri(name, *rest) # :nodoc:
uri = URI::Generic === name ? name : URI.parse(name)
mode, perm, rest = OpenURI.scan_open_optional_arguments(*rest)
options = rest.shift if !rest.empty? && Hash === rest.first
raise ArgumentError.new("extra arguments") if !rest.empty?
OpenURI.check_options(options)
mode == 'r' || mode == 'rb' ||
raise ArgumentError.new("invalid access mode #{mode} (#{uri.class} resource is read only.)")
io = open_loop(uri, options)
def OpenURI.open_loop(uri, options) # :nodoc:
case opt_proxy = options.fetch(:proxy, true)
find_proxy = lambda {|u| u.find_proxy}
find_proxy = lambda {|u| nil}
opt_proxy = URI.parse(opt_proxy)
find_proxy = lambda {|u| opt_proxy}
find_proxy = lambda {|u| opt_proxy}
raise ArgumentError.new("Invalid proxy option: #{opt_proxy}")
redirect = catch(:open_uri_redirect) {
uri.buffer_open(buf, find_proxy.call(uri), options)
# Although it violates RFC2616, Location: field may have relative
# URI. It is converted to absolute URI using uri as a base URI.
redirect = uri + redirect
unless OpenURI.redirectable?(uri, redirect)
raise "redirection forbidden: #{uri} -> #{redirect}"
if options.include? :http_basic_authentication
# send authentication only for the URI directly specified.
options.delete :http_basic_authentication
raise "HTTP redirection loop: #{uri}" if uri_set.include? uri.to_s
def OpenURI.redirectable?(uri1, uri2) # :nodoc:
# This test is intended to forbid a redirection from http://... to
# However this is ad hoc. It should be extensible/configurable.
uri1.scheme.downcase == uri2.scheme.downcase ||
(/\A(?:http|ftp)\z/i =~ uri1.scheme && /\A(?:http|ftp)\z/i =~ uri2.scheme)
def OpenURI.open_http(buf, target, proxy, options) # :nodoc:
raise "Non-HTTP proxy URI: #{proxy}" if proxy.class != URI::HTTP
if target.userinfo && "1.9.0" <= RUBY_VERSION
# don't raise for 1.8 because compatibility.
raise ArgumentError, "userinfo not supported. [RFC3986]"
klass = Net::HTTP::Proxy(proxy.host, proxy.port)
target_host = target.host
target_port = target.port
request_uri = target.request_uri
request_uri = target.to_s
http = klass.new(target_host, target_port)
if target.class == URI::HTTPS
http.verify_mode = OpenSSL::SSL::VERIFY_PEER
store = OpenSSL::X509::Store.new
options.each {|k, v| header[k] = v if String === k }
req = Net::HTTP::Get.new(request_uri, header)
if options.include? :http_basic_authentication
user, pass = options[:http_basic_authentication]
req.basic_auth user, pass
http.request(req) {|response|
if options[:content_length_proc] && Net::HTTPSuccess === resp
if resp.key?('Content-Length')
options[:content_length_proc].call(resp['Content-Length'].to_i)
options[:content_length_proc].call(nil)
if options[:progress_proc] && Net::HTTPSuccess === resp
options[:progress_proc].call(buf.size)
io.status = [resp.code, resp.message]
resp.each {|name,value| buf.io.meta_add_field name, value }
when Net::HTTPMovedPermanently, # 301
Net::HTTPTemporaryRedirect # 307
throw :open_uri_redirect, URI.parse(resp['location'])
raise OpenURI::HTTPError.new(io.status.join(' '), io)
class HTTPError < StandardError
def initialize(message, io)
if StringIO === @io && StringMax < @size
io = Tempfile.new('open-uri')
Meta.init io, @io if @io.respond_to? :meta
Meta.init @io unless @io.respond_to? :meta
# Mixin for holding meta-information.
def Meta.init(obj, src=nil) # :nodoc:
obj.base_uri = src.base_uri
src.meta.each {|name, value|
obj.meta_add_field(name, value)
# returns an Array which consists status code and message.
# returns a URI which is base of relative URIs in the data.
# It may differ from the URI supplied by a user because redirection.
# returns a Hash which represents header fields.
# The Hash keys are downcased for canonicalization.
def meta_add_field(name, value) # :nodoc:
@meta[name.downcase] = value
# returns a Time which represents Last-Modified field.
if v = @meta['last-modified']
RE_TOKEN = %r{[^\x00- ()<>@,;:\\"/\[\]?={}\x7f]+}n
RE_QUOTED_STRING = %r{"(?:[\r\n\t !#-\[\]-~\x80-\xff]|\\[\x00-\x7f])*"}n
RE_PARAMETERS = %r{(?:;#{RE_LWS}?#{RE_TOKEN}#{RE_LWS}?=#{RE_LWS}?(?:#{RE_TOKEN}|#{RE_QUOTED_STRING})#{RE_LWS}?)*}n
def content_type_parse # :nodoc:
v = @meta['content-type']
# The last (?:;#{RE_LWS}?)? matches extra ";" which violates RFC2045.
if v && %r{\A#{RE_LWS}?(#{RE_TOKEN})#{RE_LWS}?/(#{RE_TOKEN})#{RE_LWS}?(#{RE_PARAMETERS})(?:;#{RE_LWS}?)?\z}no =~ v
$3.scan(/;#{RE_LWS}?(#{RE_TOKEN})#{RE_LWS}?=#{RE_LWS}?(?:(#{RE_TOKEN})|(#{RE_QUOTED_STRING}))/no) {|att, val, qval|
val = qval.gsub(/[\r\n\t !#-\[\]-~\x80-\xff]+|(\\[\x00-\x7f])/) { $1 ? $1[1,1] : $& } if qval
parameters << [att.downcase, val]
["#{type}/#{subtype}", *parameters]
# returns "type/subtype" which is MIME Content-Type.
# It is downcased for canonicalization.
# Content-Type parameters are stripped.
type, *parameters = content_type_parse
type || 'application/octet-stream'
# returns a charset parameter in Content-Type field.
# It is downcased for canonicalization.
# If charset parameter is not given but a block is given,
# the block is called and its result is returned.
# It can be used to guess charset.
# If charset parameter and block is not given,
# nil is returned except text type in HTTP.
# In that case, "iso-8859-1" is returned as defined by RFC2616 3.7.1.
type, *parameters = content_type_parse
if pair = parameters.assoc('charset')
elsif type && %r{\Atext/} =~ type &&
@base_uri && /\Ahttp\z/i =~ @base_uri.scheme
"iso-8859-1" # RFC2616 3.7.1
# returns a list of encodings in Content-Encoding field
# The encodings are downcased for canonicalization.
v = @meta['content-encoding']
if v && %r{\A#{RE_LWS}?#{RE_TOKEN}#{RE_LWS}?(?:,#{RE_LWS}?#{RE_TOKEN}#{RE_LWS}?)*}o =~ v
v.scan(RE_TOKEN).map {|content_coding| content_coding.downcase}
# Mixin for HTTP and FTP URIs.
# OpenURI::OpenRead#open provides `open' for URI::HTTP and URI::FTP.
# OpenURI::OpenRead#open takes optional 3 arguments as:
# OpenURI::OpenRead#open([mode [, perm]] [, options]) [{|io| ... }]
# `mode', `perm' is same as Kernel#open.
# However, `mode' must be read mode because OpenURI::OpenRead#open doesn't
# support write mode (yet).
# Also `perm' is just ignored because it is meaningful only for file
# `options' must be a hash.
# Each pairs which key is a string in the hash specify a extra header
# I.e. it is ignored for FTP without HTTP proxy.
# The hash may include other options which key is a symbol:
# :proxy => "http://proxy.foo.com:8000/"
# :proxy => URI.parse("http://proxy.foo.com:8000/")
# If :proxy option is specified, the value should be String, URI,
# When String or URI is given, it is treated as proxy URI.
# When true is given or the option itself is not specified,
# environment variable `scheme_proxy' is examined.
# `scheme' is replaced by `http', `https' or `ftp'.
# When false or nil is given, the environment variables are ignored and
# connection will be made to a server directly.
# [:http_basic_authentication]
# :http_basic_authentication=>[user, password]
# If :http_basic_authentication is specified,
# the value should be an array which contains 2 strings:
# It is used for HTTP Basic authentication defined by RFC 2617.
# :content_length_proc => lambda {|content_length| ... }
# If :content_length_proc option is specified, the option value procedure
# is called before actual transfer is started.
# It takes one argument which is expected content length in bytes.
# If two or more transfer is done by HTTP redirection, the procedure
# is called only one for a last transfer.
# When expected content length is unknown, the procedure is called with
# It is happen when HTTP response has no Content-Length header.
# :progress_proc => lambda {|size| ...}
# If :progress_proc option is specified, the proc is called with one
# argument each time when `open' gets content fragment from network.
# The argument `size' `size' is a accumulated transfered size in bytes.
# If two or more transfer is done by HTTP redirection, the procedure
# is called only one for a last transfer.
# :progress_proc and :content_length_proc are intended to be used for
# For example, it can be implemented as follows using Ruby/ProgressBar.
It is recommended that you Edit text format, this type of Fix handles quite a lot in one request
