module Fog::AWS::Storage::Utils

Attributes

max_copy_chunk_size[R]
max_put_chunk_size[R]

Amazon S3 limits max chunk size that can be uploaded/copied in a single request to 5GB. Other S3-compatible storages (like, Ceph) do not have such limit. Ceph shows much better performance when file is copied as a whole, in a single request. fog-aws user can use these settings to configure chunk sizes. A non-positive value will tell fog-aws to use a single put/copy request regardless of file size.

@return [Integer] @see docs.aws.amazon.com/AmazonS3/latest/userguide/copy-object.html

region[RW]

Public Instance Methods

cdn() click to toggle source
# File lib/fog/aws/storage.rb, line 134
def cdn
  @cdn ||= Fog::AWS::CDN.new(
    :aws_access_key_id => @aws_access_key_id,
    :aws_secret_access_key => @aws_secret_access_key,
    :use_iam_profile => @use_iam_profile
  )
end
http_url(params, expires) click to toggle source
# File lib/fog/aws/storage.rb, line 142
def http_url(params, expires)
  signed_url(params.merge(:scheme => 'http'), expires)
end
https_url(params, expires) click to toggle source
# File lib/fog/aws/storage.rb, line 146
def https_url(params, expires)
  signed_url(params.merge(:scheme => 'https'), expires)
end
request_url(params) click to toggle source
# File lib/fog/aws/storage.rb, line 169
def request_url(params)
  params = request_params(params)
  params_to_url(params)
end
require_mime_types() click to toggle source
# File lib/fog/aws/storage.rb, line 155
def require_mime_types
  begin
    # Use mime/types/columnar if available, for reduced memory usage
    require 'mime/types/columnar'
  rescue LoadError
    begin
      require 'mime/types'
    rescue LoadError
      Fog::Logger.warning("'mime-types' missing, please install and try again.")
      exit(1)
    end
  end
end
signed_url(params, expires) click to toggle source
# File lib/fog/aws/storage.rb, line 174
def signed_url(params, expires)
  refresh_credentials_if_expired

  #convert expires from a point in time to a delta to now
  expires = expires.to_i
  if @signature_version == 4
    params = v4_signed_params_for_url(params, expires)
  else
    params = v2_signed_params_for_url(params, expires)
  end

  params_to_url(params)
end
url(params, expires) click to toggle source
# File lib/fog/aws/storage.rb, line 150
def url(params, expires)
  Fog::Logger.deprecation("Fog::AWS::Storage => #url is deprecated, use #https_url instead [light_black](#{caller.first})[/]")
  https_url(params, expires)
end
validate_chunk_size(value, description) click to toggle source

@param value [int] @param description [str]

# File lib/fog/aws/storage.rb, line 190
def validate_chunk_size(value, description)
  raise "#{description} (#{value}) is less than minimum #{MIN_MULTIPART_CHUNK_SIZE}" unless value <= 0 || value >= MIN_MULTIPART_CHUNK_SIZE
end

Private Instance Methods

bucket_to_path(bucket_name, path=nil) click to toggle source
# File lib/fog/aws/storage.rb, line 269
def bucket_to_path(bucket_name, path=nil)
  "/#{escape(bucket_name.to_s)}#{path}"
end
escape(string) click to toggle source

NOTE: differs from Fog::AWS.escape by NOT escaping `/`

# File lib/fog/aws/storage.rb, line 274
def escape(string)
  string.gsub(/([^a-zA-Z0-9_.\-~\/]+)/) {
    "%" + $1.unpack("H2" * $1.bytesize).join("%").upcase
  }
end
init_max_copy_chunk_size!(options = {}) click to toggle source
# File lib/fog/aws/storage.rb, line 207
def init_max_copy_chunk_size!(options = {})
  @max_copy_chunk_size = options.fetch(:max_copy_chunk_size, MAX_SINGLE_PUT_SIZE)
  validate_chunk_size(@max_copy_chunk_size, 'max_copy_chunk_size')
end
init_max_put_chunk_size!(options = {}) click to toggle source
# File lib/fog/aws/storage.rb, line 202
def init_max_put_chunk_size!(options = {})
  @max_put_chunk_size = options.fetch(:max_put_chunk_size, MAX_SINGLE_PUT_SIZE)
  validate_chunk_size(@max_put_chunk_size, 'max_put_chunk_size')
end
object_to_path(object_name=nil) click to toggle source
# File lib/fog/aws/storage.rb, line 265
def object_to_path(object_name=nil)
  '/' + escape(object_name.to_s).gsub('%2F','/')
end
params_to_url(params) click to toggle source
# File lib/fog/aws/storage.rb, line 356
def params_to_url(params)
  query = params[:query] && params[:query].map do |key, value|
    if value
      # URL parameters need / to be escaped
      [key, Fog::AWS.escape(value.to_s)].join('=')
    else
      key
    end
  end.join('&')

  URI::Generic.build({
    :scheme => params[:scheme],
    :host   => params[:host],
    :port   => params[:port],
    :path   => params[:path],
    :query  => query,
  }).to_s
end
region_to_host(region=nil) click to toggle source
# File lib/fog/aws/storage.rb, line 254
def region_to_host(region=nil)
  case region.to_s
  when DEFAULT_REGION, ''
    's3.amazonaws.com'
  when %r{\Acn-.*}
    "s3.#{region}.amazonaws.com.cn"
  else
    "s3.#{region}.amazonaws.com"
  end
end
request_params(params) click to toggle source

Transforms things like bucket_name, object_name, region

Should yield the same result when called f*f

# File lib/fog/aws/storage.rb, line 283
def request_params(params)
  headers  = params[:headers] || {}

  if params[:scheme]
    scheme = params[:scheme]
    port   = params[:port] || DEFAULT_SCHEME_PORT[scheme]
  else
    scheme = @scheme
    port   = @port
  end
  if DEFAULT_SCHEME_PORT[scheme] == port
    port = nil
  end

  if params[:region]
    region = params[:region]
    host   = params[:host] || region_to_host(region)
  else
    region = @region       || DEFAULT_REGION
    host   = params[:host] || @host || region_to_host(region)
  end

  path     = params[:path] || object_to_path(params[:object_name])
  path     = '/' + path if path[0..0] != '/'

  if params[:bucket_name]
    bucket_name = params[:bucket_name]

    if params[:bucket_cname]
      host = bucket_name
    else
      path_style = params.fetch(:path_style, @path_style)
      if !path_style
        if COMPLIANT_BUCKET_NAMES !~ bucket_name
          Fog::Logger.warning("fog: the specified s3 bucket name(#{bucket_name}) is not a valid dns name, which will negatively impact performance.  For details see: https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html")
          path_style = true
        elsif scheme == 'https' && !path_style && bucket_name =~ /\./
          Fog::Logger.warning("fog: the specified s3 bucket name(#{bucket_name}) contains a '.' so is not accessible over https as a virtual hosted bucket, which will negatively impact performance.  For details see: https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html")
          path_style = true
        end
      end

      # uses the bucket name as host if `virtual_host: true`, you can also
      # manually specify the cname if required.
      if params[:virtual_host]
        host = params.fetch(:cname, bucket_name)
      elsif path_style
        path = bucket_to_path bucket_name, path
      elsif host.start_with?("#{bucket_name}.")
        # no-op
      else
        host = [bucket_name, host].join('.')
      end
    end
  end

  ret = params.merge({
    :scheme       => scheme,
    :host         => host,
    :port         => port,
    :path         => path,
    :headers      => headers
  })

  #
  ret.delete(:path_style)
  ret.delete(:bucket_name)
  ret.delete(:object_name)
  ret.delete(:region)

  ret
end
v2_signed_params_for_url(params, expires) click to toggle source
# File lib/fog/aws/storage.rb, line 235
def v2_signed_params_for_url(params, expires)
  if @aws_session_token
    params[:headers]||= {}
    params[:headers]['x-amz-security-token'] = @aws_session_token
  end
  signature = signature_v2(params, expires)

  params = request_params(params)

  signature_query_params = {
    'AWSAccessKeyId' => @aws_access_key_id,
    'Signature' => signature,
    'Expires' => expires,
  }
  params[:query] = (params[:query] || {}).merge(signature_query_params)
  params[:query]['x-amz-security-token'] = @aws_session_token if @aws_session_token
  params
end
v4_signed_params_for_url(params, expires) click to toggle source
# File lib/fog/aws/storage.rb, line 212
def v4_signed_params_for_url(params, expires)
  now = Fog::Time.now

  expires = expires - now.to_i
  params[:headers] ||= {}

  params[:query]||= {}
  params[:query]['X-Amz-Expires'] = expires
  params[:query]['X-Amz-Date'] = now.to_iso8601_basic

  if @aws_session_token
    params[:query]['X-Amz-Security-Token'] = @aws_session_token
  end

  params = request_params(params)
  params[:headers][:host] = params[:host]
  params[:headers][:host] += ":#{params[:port]}" if params.fetch(:port, nil)

  signature_query_params = @signer.signature_parameters(params, now, "UNSIGNED-PAYLOAD")
  params[:query] = (params[:query] || {}).merge(signature_query_params)
  params
end
validate_signature_version!() click to toggle source
# File lib/fog/aws/storage.rb, line 196
def validate_signature_version!
  unless @signature_version == 2 || @signature_version == 4
    raise "Unknown signature version #{@signature_version}; valid versions are 2 or 4"
  end
end