class Fog::Storage::GoogleJSON::Real
Attributes
Public Class Methods
# File lib/fog/storage/google_json/real.rb, line 11 def initialize(options = {}) shared_initialize(options[:google_project], GOOGLE_STORAGE_JSON_API_VERSION, GOOGLE_STORAGE_JSON_BASE_URL) options[:google_api_scope_url] = GOOGLE_STORAGE_JSON_API_SCOPE_URLS.join(" ") @host = options[:host] || "storage.googleapis.com" # TODO(temikus): Do we even need this client? @client = initialize_google_client(options) # IAM client used for SignBlob API @iam_service = ::Google::Apis::IamcredentialsV1::IAMCredentialsService.new apply_client_options(@iam_service, { google_api_scope_url: GOOGLE_STORAGE_JSON_IAM_API_SCOPE_URLS.join(" ") }) @storage_json = ::Google::Apis::StorageV1::StorageService.new apply_client_options(@storage_json, options) @storage_json.client_options.open_timeout_sec = options[:open_timeout_sec] if options[:open_timeout_sec] @storage_json.client_options.read_timeout_sec = options[:read_timeout_sec] if options[:read_timeout_sec] @storage_json.client_options.send_timeout_sec = options[:send_timeout_sec] if options[:send_timeout_sec] end
Public Instance Methods
Copy an object from one Google Storage bucket to another
@param source_bucket [String] Name of source bucket @param source_object [String] Name of source object @param target_bucket [String] Name of bucket to create copy in @param target_object [String] Name of new copy of object
@see cloud.google.com/storage/docs/json_api/v1/objects/copy @return [Google::Apis::StorageV1::Object] copy of object
# File lib/fog/storage/google_json/requests/copy_object.rb, line 14 def copy_object(source_bucket, source_object, target_bucket, target_object, options = {}) request_options = ::Google::Apis::RequestOptions.default.merge(options) @storage_json.copy_object(source_bucket, source_object, target_bucket, target_object, request_options) end
Delete an Google Storage bucket cloud.google.com/storage/docs/json_api/v1/buckets/delete
@param bucket_name [String] Name of bucket to delete
# File lib/fog/storage/google_json/requests/delete_bucket.rb, line 9 def delete_bucket(bucket_name) @storage_json.delete_bucket(bucket_name) end
Delete an object from Google Storage cloud.google.com/storage/docs/json_api/v1/objects/delete
@param bucket_name [String] Name of bucket containing object to delete @param object_name [String] Name of object to delete
# File lib/fog/storage/google_json/requests/delete_object.rb, line 10 def delete_object(bucket_name, object_name) @storage_json.delete_object(bucket_name, object_name) end
Get an expiring object url from Google Storage for deleting an object cloud.google.com/storage/docs/access-control#Signed-URLs
@param bucket_name [String] Name of bucket containing object @param object_name [String] Name of object to get expiring url for @param expires [Time] Expiry time for this URL
@return [String] Expiring object https URL
# File lib/fog/storage/google_json/requests/delete_object_url.rb, line 13 def delete_object_url(bucket_name, object_name, expires) raise ArgumentError.new("bucket_name is required") unless bucket_name raise ArgumentError.new("object_name is required") unless object_name https_url({ :headers => {}, :host => @host, :method => "DELETE", :path => "#{bucket_name}/#{object_name}" }, expires) end
List information about objects in an Google Storage bucket # cloud.google.com/storage/docs/json_api/v1/buckets#resource
@param bucket_name [String]
Name of bucket to list
@param [Fixnum] if_metageneration_match
Makes the return of the bucket metadata conditional on whether the bucket's current metageneration matches the given value.
@param [Fixnum] if_metageneration_not_match
Makes the return of the bucket metadata conditional on whether the bucket's current metageneration does not match the given value.
@param [String] projection
Set of properties to return. Defaults to noAcl.
@return [Google::Apis::StorageV1::Bucket]
# File lib/fog/storage/google_json/requests/get_bucket.rb, line 19 def get_bucket(bucket_name, if_metageneration_match: nil, if_metageneration_not_match: nil, projection: nil) raise ArgumentError.new("bucket_name is required") unless bucket_name @storage_json.get_bucket( bucket_name, :if_metageneration_match => if_metageneration_match, :if_metageneration_not_match => if_metageneration_not_match, :projection => projection ) end
Get access control list entry for an Google Storage bucket @see cloud.google.com/storage/docs/json_api/v1/bucketAccessControls/get
@param bucket_name [String]
Name of bucket
@param entity [String]
The entity holding the permission. Can be user-userId, user-emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers.
@return [Google::Apis::StorageV1::BucketAccessControls]
# File lib/fog/storage/google_json/requests/get_bucket_acl.rb, line 15 def get_bucket_acl(bucket_name, entity) raise ArgumentError.new("bucket_name is required") unless bucket_name raise ArgumentError.new("entity is required") unless entity @storage_json.get_bucket_access_control(bucket_name, entity) end
Get an object from Google Storage @see cloud.google.com/storage/docs/json_api/v1/objects/get
@param bucket_name [String] Name of bucket to create object in @param object_name [String] Name of object to create @param generation [Fixnum]
If present, selects a specific revision of this object (as opposed to the latest version, the default).
@param ifGenerationMatch [Fixnum]
Makes the operation conditional on whether the object's current generation matches the given value. Setting to 0 makes the operation succeed only if there are no live versions of the object.
@param ifGenerationNotMatch [Fixnum]
Makes the operation conditional on whether the object's current generation does not match the given value. If no live object exists, the precondition fails. Setting to 0 makes the operation succeed only if there is a live version of the object.
@param ifMetagenerationMatch [Fixnum]
Makes the operation conditional on whether the object's current metageneration matches the given value.
@param ifMetagenerationNotMatch [Fixnum]
Makes the operation conditional on whether the object's current metageneration does not match the given value.
@param projection [Fixnum]
Set of properties to return
@param options [Hash]
Request-specific options
@param &block [Proc]
Block to pass a streamed object response to. Expected format is same as Excon :response_block ({ |chunk, remaining_bytes, total_bytes| ... })
@return [Hash] Object metadata with :body attribute set to contents of object
# File lib/fog/storage/google_json/requests/get_object.rb, line 30 def get_object(bucket_name, object_name, generation: nil, if_generation_match: nil, if_generation_not_match: nil, if_metageneration_match: nil, if_metageneration_not_match: nil, projection: nil, **options, &_block) raise ArgumentError.new("bucket_name is required") unless bucket_name raise ArgumentError.new("object_name is required") unless object_name buf = Tempfile.new("fog-google-storage-temp") # Two requests are necessary, first for metadata, then for content. # google-api-ruby-client doesn't allow fetching both metadata and content request_options = ::Google::Apis::RequestOptions.default.merge(options) all_opts = { :generation => generation, :if_generation_match => if_generation_match, :if_generation_not_match => if_generation_not_match, :if_metageneration_match => if_metageneration_match, :if_metageneration_not_match => if_metageneration_not_match, :projection => projection, :options => request_options } object = @storage_json.get_object(bucket_name, object_name, all_opts).to_h @storage_json.get_object( bucket_name, object_name, all_opts.merge(:download_dest => buf.path) ) if block_given? yield buf.read, nil, nil else object[:body] = buf.read buf.unlink end object ensure buf.close! rescue nil end
Get access control list for an Google Storage object cloud.google.com/storage/docs/json_api/v1/objectAccessControls/get
@param bucket_name [String] Name of bucket object is in @param object_name [String] Name of object to add ACL to @param entity [String] The entity holding the permission.
Can be user-userId, user-emailAddress, group-groupId, group-emailAddress, allUsers, or allAuthenticatedUsers.
@param generation [Hash] Specify a particular version to retrieve @return [Google::Apis::StorageV1::ObjectAccessControls]
# File lib/fog/storage/google_json/requests/get_object_acl.rb, line 15 def get_object_acl(bucket_name, object_name, entity, generation: nil) raise ArgumentError.new("bucket_name is required") unless bucket_name raise ArgumentError.new("object_name is required") unless object_name @storage_json.get_object_access_control( bucket_name, object_name, entity, :generation => generation ) end
Fetch metadata for an object in Google Storage
@param bucket_name [String] Name of bucket to read from @param object_name [String] Name of object to read @param options [Hash] Optional parameters @see cloud.google.com/storage/docs/json_api/v1/objects/get
@return [Google::Apis::StorageV1::Object]
# File lib/fog/storage/google_json/requests/get_object_metadata.rb, line 13 def get_object_metadata(bucket_name, object_name, options = {}) raise ArgumentError.new("bucket_name is required") unless bucket_name raise ArgumentError.new("object_name is required") unless object_name request_options = ::Google::Apis::RequestOptions.default.merge(options) @storage_json.get_object(bucket_name, object_name, :options => request_options) end
Get an expiring object url from GCS Deprecated, redirects to get_object_https_url.rb
# File lib/fog/storage/google_json/requests/get_object_url.rb, line 7 def get_object_url(bucket_name, object_name, expires, options = {}) Fog::Logger.deprecation("Fog::Storage::Google => #get_object_url is deprecated, use #get_object_https_url instead[/] [light_black](#{caller(0..0)})") get_object_https_url(bucket_name, object_name, expires, options) end
Get access control list for an Google Storage bucket @see cloud.google.com/storage/docs/json_api/v1/bucketAccessControls/list
@param bucket_name [String] Name of bucket object is in @return [Google::Apis::StorageV1::BucketAccessControls]
# File lib/fog/storage/google_json/requests/list_bucket_acl.rb, line 10 def list_bucket_acl(bucket_name) raise ArgumentError.new("bucket_name is required") unless bucket_name @storage_json.list_bucket_access_controls(bucket_name) end
Retrieves a list of buckets for a given project cloud.google.com/storage/docs/json_api/v1/buckets/list
@return [Google::Apis::StorageV1::Buckets] TODO: check if very large lists require working with nextPageToken
# File lib/fog/storage/google_json/requests/list_buckets.rb, line 10 def list_buckets(max_results: nil, page_token: nil, prefix: nil, projection: nil) @storage_json.list_buckets( @project, :max_results => max_results, :page_token => page_token, :prefix => prefix, :projection => projection ) end
List access control list for an Google Storage object cloud.google.com/storage/docs/json_api/v1/objectAccessControls/get
@param bucket_name [String] Name of bucket object is in @param object_name [String] Name of object to add ACL to @return [Google::Apis::StorageV1::ObjectAccessControls]
# File lib/fog/storage/google_json/requests/list_object_acl.rb, line 11 def list_object_acl(bucket_name, object_name, generation: nil) raise ArgumentError.new("bucket_name is required") unless bucket_name raise ArgumentError.new("object_name is required") unless object_name @storage_json.list_object_access_controls(bucket_name, object_name, :generation => generation) end
Lists objects in a bucket matching some criteria.
@param bucket [String] Name of bucket to list @param options [Hash] Optional hash of options @option options [String] :delimiter Delimiter to collapse objects
under to emulate a directory-like mode
@option options [Integer] :max_results Maximum number of results to
retrieve
@option options [String] :page_token Token to select a particular page
of results
@option options [String] :prefix String that an object must begin with
in order to be returned
@option options [“full”, “noAcl”] :projection Set of properties to
return (defaults to "noAcl")
@option options [Boolean] :versions If true, lists all versions of an
object as distinct results (defaults to False)
@return [Google::Apis::StorageV1::Objects]
# File lib/fog/storage/google_json/requests/list_objects.rb, line 22 def list_objects(bucket, options = {}) allowed_opts = %i( delimiter max_results page_token prefix projection versions ) @storage_json.list_objects( bucket, options.select { |k, _| allowed_opts.include? k } ) end
Create a Google Storage bucket @see cloud.google.com/storage/docs/json_api/v1/buckets/insert
@param bucket_name [String] Name of bucket to create @param options [Hash]
Optional fields. Acceptable options include any writeable bucket attribute (see docs) or one of the following options:
@param predefined_acl [String] Applies a predefined set of access controls to this bucket. @param predefined_default_object_acl [String] Applies a predefined set of default object access controls
@return [Google::Apis::StorageV1::Bucket]
# File lib/fog/storage/google_json/requests/put_bucket.rb, line 17 def put_bucket(bucket_name, predefined_acl: nil, predefined_default_object_acl: nil, **options) bucket = ::Google::Apis::StorageV1::Bucket.new( options.merge(:name => bucket_name) ) @storage_json.insert_bucket( @project, bucket, :predefined_acl => predefined_acl, :predefined_default_object_acl => predefined_default_object_acl ) end
Change access control list for an Google Storage bucket cloud.google.com/storage/docs/json_api/v1/bucketAccessControls/insert
@param bucket_name [String] Name of bucket object is in @param acl [Hash] ACL hash to add to bucket, see GCS documentation above @return [Google::Apis::StorageV1::BucketAccessControl]
# File lib/fog/storage/google_json/requests/put_bucket_acl.rb, line 11 def put_bucket_acl(bucket_name, acl = {}) raise ArgumentError.new("bucket_name is required") unless bucket_name raise ArgumentError.new("acl is required") unless acl acl_update = ::Google::Apis::StorageV1::BucketAccessControl.new(acl) @storage_json.insert_bucket_access_control(bucket_name, acl_update) end
Create an object in an Google Storage bucket cloud.google.com/storage/docs/json_api/v1/objects/insert
@param bucket_name [String] Name of bucket to create object in @param object_name [String] Name of object to create @param data [File|String|Paperclip::AbstractAdapter] File, String or Paperclip adapter to create object from @param options [Hash] Optional query parameters or Object attributes
Optional query parameters are listed below.
@param content_encoding [String]
If set, sets the contentEncoding property of the final object to this value.
@param if_generation_match [Fixnum]
Makes the operation conditional on whether the object's current generation matches the given value. Setting to 0 makes the operation succeed only if there are no live versions of the object.
@param if_generation_not_match [Fixnum]
Makes the operation conditional on whether the object's current generation does not match the given value. If no live object exists, the precondition fails. Setting to 0 makes the operation succeed only if there is a live version of the object.
@param if_metageneration_match [Fixnum]
Makes the operation conditional on whether the object's current metageneration matches the given value.
@param if_metageneration_not_match [Fixnum]
Makes the operation conditional on whether the object's current metageneration does not match the given value.
@param predefined_acl [String]
Apply a predefined set of access controls to this object.
@param projection [String]
Set of properties to return. Defaults to noAcl, unless the object resource specifies the acl property, when it defaults to full.
@return [Google::Apis::StorageV1::Object]
# File lib/fog/storage/google_json/requests/put_object.rb, line 39 def put_object(bucket_name, object_name, data, content_encoding: nil, if_generation_match: nil, if_generation_not_match: nil, if_metageneration_match: nil, if_metageneration_not_match: nil, kms_key_name: nil, predefined_acl: nil, **options) data, options = normalize_data(data, options) object_config = ::Google::Apis::StorageV1::Object.new( options.merge(:name => object_name) ) @storage_json.insert_object( bucket_name, object_config, :content_encoding => content_encoding, :if_generation_match => if_generation_match, :if_generation_not_match => if_generation_not_match, :if_metageneration_match => if_metageneration_match, :if_metageneration_not_match => if_metageneration_not_match, :kms_key_name => kms_key_name, :predefined_acl => predefined_acl, :options => ::Google::Apis::RequestOptions.default.merge(options), # see https://developers.google.com/api-client-library/ruby/guide/media_upload :content_type => options[:content_type], :upload_source => data ) end
Change access control list for an Google Storage object
@param bucket_name [String] name of bucket object is in @param object_name [String] name of object to add ACL to @param acl [Hash] ACL hash to add to bucket, see GCS documentation.
@see cloud.google.com/storage/docs/json_api/v1/objectAccessControls/insert @return [Google::Apis::StorageV1::ObjectAccessControl]
# File lib/fog/storage/google_json/requests/put_object_acl.rb, line 13 def put_object_acl(bucket_name, object_name, acl) raise ArgumentError.new("bucket_name is required") unless bucket_name raise ArgumentError.new("object_name is required") unless object_name raise ArgumentError.new("acl is required") unless acl acl_object = ::Google::Apis::StorageV1::ObjectAccessControl.new(acl) @storage_json.insert_object_access_control( bucket_name, object_name, acl_object ) end
Get an expiring object url from Google Storage for putting an object cloud.google.com/storage/docs/access-control#Signed-URLs
@param bucket_name [String] Name of bucket containing object @param object_name [String] Name of object to get expiring url for @param expires [Time] Expiry time for this URL @param headers [Hash] Optional hash of headers to include @option options [String] “x-goog-acl” Permissions, must be in ['private', 'public-read', 'public-read-write', 'authenticated-read'].
If you want a file to be public you should to add { 'x-goog-acl' => 'public-read' } to headers and then call for example: curl -H "x-goog-acl:public-read" "signed url"
@return [String] Expiring object https URL
# File lib/fog/storage/google_json/requests/put_object_url.rb, line 16 def put_object_url(bucket_name, object_name, expires, headers = {}) raise ArgumentError.new("bucket_name is required") unless bucket_name raise ArgumentError.new("object_name is required") unless object_name https_url({ :headers => headers, :host => @host, :method => "PUT", :path => "#{bucket_name}/#{object_name}" }, expires) end
# File lib/fog/storage/google_json/real.rb, line 32 def signature(params) string_to_sign = <<-DATA #{params[:method]} #{params[:headers]['Content-MD5']} #{params[:headers]['Content-Type']} #{params[:headers]['Date']} DATA google_headers = {} canonical_google_headers = "" params[:headers].each do |key, value| google_headers[key] = value if key[0..6] == "x-goog-" end google_headers = google_headers.sort_by { |a| a[0] } google_headers.each do |key, value| canonical_google_headers << "#{key}:#{value}\n" end string_to_sign << canonical_google_headers.to_s canonical_resource = "/" if subdomain = params.delete(:subdomain) canonical_resource << "#{CGI.escape(subdomain).downcase}/" end canonical_resource << params[:path].to_s canonical_resource << "?" (params[:query] || {}).each_key do |key| if %w(acl cors location logging requestPayment versions versioning).include?(key) canonical_resource << "#{key}&" end end canonical_resource.chop! string_to_sign << canonical_resource.to_s # TODO(temikus): make signer configurable or add ability to supply your own via lambda if !@storage_json.authorization.signing_key.nil? signed_string = default_signer(string_to_sign) else # If client doesn't contain signing key attempt to auth via IAM SignBlob API signed_string = iam_signer(string_to_sign) end Base64.encode64(signed_string).chomp! end
Protected Instance Methods
# File lib/fog/storage/google_json/requests/put_object.rb, line 74 def normalize_data(data, options) raise ArgumentError.new("data is required") unless data if data.is_a?(String) data = StringIO.new(data) options[:content_type] ||= "text/plain" elsif data.is_a?(::File) options[:content_type] ||= Fog::Storage.parse_data(data)[:headers]["Content-Type"] end # Paperclip::AbstractAdapter if data.respond_to?(:content_type) && data.respond_to?(:path) options[:content_type] ||= data.content_type data = data.path end [data, options] end
Private Instance Methods
Default url signer using service account keys
@param [String] string_to_sign Special collection of headers and options for V2 storage signing, e.g.:
StringToSign = HTTP_Verb + "\n" + Content_MD5 + "\n" + Content_Type + "\n" + Expires + "\n" + Canonicalized_Extension_Headers + Canonicalized_Resource See https://cloud.google.com/storage/docs/access-control/signed-urls-v2
@return [String] Signature binary blob
# File lib/fog/storage/google_json/real.rb, line 137 def default_signer(string_to_sign) key = OpenSSL::PKey::RSA.new(@storage_json.authorization.signing_key) digest = OpenSSL::Digest::SHA256.new return key.sign(digest, string_to_sign) end
Attempts to fetch the google service account name from metadata using Google::Cloud::Env
@return [String] Service account name, typically needed for GoogleAccessId, e.g.
my-account@project.iam.gserviceaccount
@raises [Fog::Errors::Error] If Metadata service is not available or returns an invalid response
# File lib/fog/storage/google_json/real.rb, line 108 def get_access_id_from_metadata if @google_cloud_env.metadata? access_id = @google_cloud_env.lookup_metadata("instance", "service-accounts/default/email") else raise Fog::Errors::Error.new("Metadata service not available, unable to retrieve service account info.") end if access_id.nil? raise Fog::Errors::Error.new("Metadata service found but didn't return data." \ "Please file a bug: https://github.com/fog/fog-google") end return access_id end
Fetches the google service account name
@return [String] Service account name, typically needed for GoogleAccessId, e.g.
my-account@project.iam.gserviceaccount
@raises [Fog::Errors::Error] If authorisation is incorrect or inapplicable for current action
# File lib/fog/storage/google_json/real.rb, line 89 def get_google_access_id if @storage_json.authorization.is_a?(::Google::Auth::UserRefreshCredentials) raise Fog::Errors::Error.new("User / Application Default Credentials are not supported for storage"\ "url signing, please use a service account or metadata authentication.") end if !@storage_json.authorization.issuer.nil? return @storage_json.authorization.issuer else get_access_id_from_metadata end end
# File lib/fog/storage/google_json/real.rb, line 79 def google_access_id @google_access_id ||= get_google_access_id end
Fallback URL signer using the IAM SignServiceAccountBlob API, see
Google::Apis::IamcredentialsV1::IAMCredentialsService#sign_service_account_blob
@param [String] string_to_sign Special collection of headers and options for V2 storage signing, e.g.:
StringToSign = HTTP_Verb + "\n" + Content_MD5 + "\n" + Content_Type + "\n" + Expires + "\n" + Canonicalized_Extension_Headers + Canonicalized_Resource See https://cloud.google.com/storage/docs/access-control/signed-urls-v2
@return [String] Signature binary blob
# File lib/fog/storage/google_json/real.rb, line 158 def iam_signer(string_to_sign) request = { "payload": string_to_sign } resource = "projects/-/serviceAccounts/#{google_access_id}" response = @iam_service.sign_service_account_blob resource, request, {} return response.signed_blob end