module Fog::AWS::Storage::Utils
Attributes
max_copy_chunk_size[R]
max_put_chunk_size[R]
Amazon S3 limits max chunk size that can be uploaded/copied in a single request to 5GB. Other S3-compatible storages (like, Ceph) do not have such limit. Ceph shows much better performance when file is copied as a whole, in a single request. fog-aws user can use these settings to configure chunk sizes. A non-positive value will tell fog-aws to use a single put/copy request regardless of file size.
@return [Integer] @see docs.aws.amazon.com/AmazonS3/latest/userguide/copy-object.html
region[RW]
Public Instance Methods
cdn()
click to toggle source
# File lib/fog/aws/storage.rb, line 134 def cdn @cdn ||= Fog::AWS::CDN.new( :aws_access_key_id => @aws_access_key_id, :aws_secret_access_key => @aws_secret_access_key, :use_iam_profile => @use_iam_profile ) end
http_url(params, expires)
click to toggle source
# File lib/fog/aws/storage.rb, line 142 def http_url(params, expires) signed_url(params.merge(:scheme => 'http'), expires) end
https_url(params, expires)
click to toggle source
# File lib/fog/aws/storage.rb, line 146 def https_url(params, expires) signed_url(params.merge(:scheme => 'https'), expires) end
request_url(params)
click to toggle source
# File lib/fog/aws/storage.rb, line 169 def request_url(params) params = request_params(params) params_to_url(params) end
require_mime_types()
click to toggle source
# File lib/fog/aws/storage.rb, line 155 def require_mime_types begin # Use mime/types/columnar if available, for reduced memory usage require 'mime/types/columnar' rescue LoadError begin require 'mime/types' rescue LoadError Fog::Logger.warning("'mime-types' missing, please install and try again.") exit(1) end end end
signed_url(params, expires)
click to toggle source
# File lib/fog/aws/storage.rb, line 174 def signed_url(params, expires) refresh_credentials_if_expired #convert expires from a point in time to a delta to now expires = expires.to_i if @signature_version == 4 params = v4_signed_params_for_url(params, expires) else params = v2_signed_params_for_url(params, expires) end params_to_url(params) end
url(params, expires)
click to toggle source
# File lib/fog/aws/storage.rb, line 150 def url(params, expires) Fog::Logger.deprecation("Fog::AWS::Storage => #url is deprecated, use #https_url instead [light_black](#{caller.first})[/]") https_url(params, expires) end
validate_chunk_size(value, description)
click to toggle source
@param value [int] @param description [str]
# File lib/fog/aws/storage.rb, line 190 def validate_chunk_size(value, description) raise "#{description} (#{value}) is less than minimum #{MIN_MULTIPART_CHUNK_SIZE}" unless value <= 0 || value >= MIN_MULTIPART_CHUNK_SIZE end
Private Instance Methods
bucket_to_path(bucket_name, path=nil)
click to toggle source
# File lib/fog/aws/storage.rb, line 269 def bucket_to_path(bucket_name, path=nil) "/#{escape(bucket_name.to_s)}#{path}" end
escape(string)
click to toggle source
NOTE: differs from Fog::AWS.escape
by NOT escaping `/`
# File lib/fog/aws/storage.rb, line 274 def escape(string) string.gsub(/([^a-zA-Z0-9_.\-~\/]+)/) { "%" + $1.unpack("H2" * $1.bytesize).join("%").upcase } end
init_max_copy_chunk_size!(options = {})
click to toggle source
# File lib/fog/aws/storage.rb, line 207 def init_max_copy_chunk_size!(options = {}) @max_copy_chunk_size = options.fetch(:max_copy_chunk_size, MAX_SINGLE_PUT_SIZE) validate_chunk_size(@max_copy_chunk_size, 'max_copy_chunk_size') end
init_max_put_chunk_size!(options = {})
click to toggle source
# File lib/fog/aws/storage.rb, line 202 def init_max_put_chunk_size!(options = {}) @max_put_chunk_size = options.fetch(:max_put_chunk_size, MAX_SINGLE_PUT_SIZE) validate_chunk_size(@max_put_chunk_size, 'max_put_chunk_size') end
object_to_path(object_name=nil)
click to toggle source
# File lib/fog/aws/storage.rb, line 265 def object_to_path(object_name=nil) '/' + escape(object_name.to_s).gsub('%2F','/') end
params_to_url(params)
click to toggle source
# File lib/fog/aws/storage.rb, line 356 def params_to_url(params) query = params[:query] && params[:query].map do |key, value| if value # URL parameters need / to be escaped [key, Fog::AWS.escape(value.to_s)].join('=') else key end end.join('&') URI::Generic.build({ :scheme => params[:scheme], :host => params[:host], :port => params[:port], :path => params[:path], :query => query, }).to_s end
region_to_host(region=nil)
click to toggle source
# File lib/fog/aws/storage.rb, line 254 def region_to_host(region=nil) case region.to_s when DEFAULT_REGION, '' 's3.amazonaws.com' when %r{\Acn-.*} "s3.#{region}.amazonaws.com.cn" else "s3.#{region}.amazonaws.com" end end
request_params(params)
click to toggle source
Transforms things like bucket_name, object_name, region
Should yield the same result when called f*f
# File lib/fog/aws/storage.rb, line 283 def request_params(params) headers = params[:headers] || {} if params[:scheme] scheme = params[:scheme] port = params[:port] || DEFAULT_SCHEME_PORT[scheme] else scheme = @scheme port = @port end if DEFAULT_SCHEME_PORT[scheme] == port port = nil end if params[:region] region = params[:region] host = params[:host] || region_to_host(region) else region = @region || DEFAULT_REGION host = params[:host] || @host || region_to_host(region) end path = params[:path] || object_to_path(params[:object_name]) path = '/' + path if path[0..0] != '/' if params[:bucket_name] bucket_name = params[:bucket_name] if params[:bucket_cname] host = bucket_name else path_style = params.fetch(:path_style, @path_style) if !path_style if COMPLIANT_BUCKET_NAMES !~ bucket_name Fog::Logger.warning("fog: the specified s3 bucket name(#{bucket_name}) is not a valid dns name, which will negatively impact performance. For details see: https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html") path_style = true elsif scheme == 'https' && !path_style && bucket_name =~ /\./ Fog::Logger.warning("fog: the specified s3 bucket name(#{bucket_name}) contains a '.' so is not accessible over https as a virtual hosted bucket, which will negatively impact performance. For details see: https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html") path_style = true end end # uses the bucket name as host if `virtual_host: true`, you can also # manually specify the cname if required. if params[:virtual_host] host = params.fetch(:cname, bucket_name) elsif path_style path = bucket_to_path bucket_name, path elsif host.start_with?("#{bucket_name}.") # no-op else host = [bucket_name, host].join('.') end end end ret = params.merge({ :scheme => scheme, :host => host, :port => port, :path => path, :headers => headers }) # ret.delete(:path_style) ret.delete(:bucket_name) ret.delete(:object_name) ret.delete(:region) ret end
v2_signed_params_for_url(params, expires)
click to toggle source
# File lib/fog/aws/storage.rb, line 235 def v2_signed_params_for_url(params, expires) if @aws_session_token params[:headers]||= {} params[:headers]['x-amz-security-token'] = @aws_session_token end signature = signature_v2(params, expires) params = request_params(params) signature_query_params = { 'AWSAccessKeyId' => @aws_access_key_id, 'Signature' => signature, 'Expires' => expires, } params[:query] = (params[:query] || {}).merge(signature_query_params) params[:query]['x-amz-security-token'] = @aws_session_token if @aws_session_token params end
v4_signed_params_for_url(params, expires)
click to toggle source
# File lib/fog/aws/storage.rb, line 212 def v4_signed_params_for_url(params, expires) now = Fog::Time.now expires = expires - now.to_i params[:headers] ||= {} params[:query]||= {} params[:query]['X-Amz-Expires'] = expires params[:query]['X-Amz-Date'] = now.to_iso8601_basic if @aws_session_token params[:query]['X-Amz-Security-Token'] = @aws_session_token end params = request_params(params) params[:headers][:host] = params[:host] params[:headers][:host] += ":#{params[:port]}" if params.fetch(:port, nil) signature_query_params = @signer.signature_parameters(params, now, "UNSIGNED-PAYLOAD") params[:query] = (params[:query] || {}).merge(signature_query_params) params end
validate_signature_version!()
click to toggle source
# File lib/fog/aws/storage.rb, line 196 def validate_signature_version! unless @signature_version == 2 || @signature_version == 4 raise "Unknown signature version #{@signature_version}; valid versions are 2 or 4" end end