class Down::Wget
Public Class Methods
new(*arguments)
click to toggle source
# File lib/down/wget.rb, line 17 def initialize(*arguments) @arguments = [ user_agent: "Down/#{Down::VERSION}", max_redirect: 2, dns_timeout: 30, connect_timeout: 30, read_timeout: 30, ] + arguments end
Public Instance Methods
download(url, *args, max_size: nil, content_length_proc: nil, progress_proc: nil, destination: nil, **options)
click to toggle source
# File lib/down/wget.rb, line 27 def download(url, *args, max_size: nil, content_length_proc: nil, progress_proc: nil, destination: nil, **options) io = open(url, *args, **options, rewindable: false) content_length_proc.call(io.size) if content_length_proc && io.size if max_size && io.size && io.size > max_size raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)" end extname = File.extname(URI(url).path) tempfile = Tempfile.new(["down-wget", extname], binmode: true) until io.eof? chunk = io.readpartial(nil, buffer ||= String.new) tempfile.write(chunk) progress_proc.call(tempfile.size) if progress_proc if max_size && tempfile.size > max_size raise Down::TooLarge, "file is too large (max is #{max_size/1024/1024}MB)" end end tempfile.open # flush written content tempfile.extend Down::Wget::DownloadedFile tempfile.url = url tempfile.headers = io.data[:headers] download_result(tempfile, destination) rescue tempfile.close! if tempfile raise ensure io.close if io end
open(url, *args, rewindable: true, **options)
click to toggle source
# File lib/down/wget.rb, line 65 def open(url, *args, rewindable: true, **options) arguments = generate_command(url, *args, **options) command = Down::Wget::Command.execute(arguments) output = Down::ChunkedIO.new( chunks: command.enum_for(:output), on_close: command.method(:terminate), rewindable: false, ) # https://github.com/tmm1/http_parser.rb/issues/29#issuecomment-309976363 header_string = output.readpartial header_string << output.readpartial until header_string.include?("\r\n\r\n") header_string, first_chunk = header_string.split("\r\n\r\n", 2) parser = HTTP::Parser.new parser << header_string if parser.headers.nil? output.close raise Down::Error, "failed to parse response headers" end headers = parser.headers status = parser.status_code content_length = headers["Content-Length"].to_i if headers["Content-Length"] charset = headers["Content-Type"][/;\s*charset=([^;]+)/i, 1] if headers["Content-Type"] chunks = Enumerator.new do |yielder| yielder << first_chunk if first_chunk yielder << output.readpartial until output.eof? end Down::ChunkedIO.new( chunks: chunks, size: content_length, encoding: charset, rewindable: rewindable, on_close: output.method(:close), data: { status: status, headers: headers }, ) end
Private Instance Methods
generate_command(url, *args, **options)
click to toggle source
# File lib/down/wget.rb, line 111 def generate_command(url, *args, **options) command = %W[wget --no-verbose --save-headers -O -] options = @arguments.grep(Hash).inject({}, :merge).merge(options) args = @arguments.grep(->(o){!o.is_a?(Hash)}) + args (args + options.to_a).each do |option, value| if option.is_a?(String) command << option elsif option.length == 1 command << "-#{option}" else command << "--#{option.to_s.gsub("_", "-")}" end command << value.to_s unless value.nil? end command << url command end