From 09df67aef75b92e88b4674feb2d35cc22dc5d44e Mon Sep 17 00:00:00 2001 From: wb-lhx292136 Date: Sun, 8 Apr 2018 16:22:30 +0800 Subject: [PATCH 1/6] update put-object interface for multi-thread scenario. --- README.md | 8 +++--- lib/emulator/cmdline.rb | 4 +++ lib/emulator/logging.rb | 1 + lib/emulator/multipart.rb | 14 ++++++---- lib/emulator/object.rb | 24 ++++++++++++----- lib/emulator/util.rb | 54 ++++++++++++++++++++++++++++++++++++--- 6 files changed, 87 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index ce62793..90df667 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ UploadPartCopy,AbortMultipartUpload,ListMultipartUpload,ListParts - 下载 [oss-emulator](https://github.com/aliyun/oss-emulator) -- 运行。进入 *oss-emulator* 目录, 执行命令 `ruby bin/emulator -r store -p 8080`。 +- 运行。进入 *oss-emulator* 目录, 执行命令 `ruby bin/emulator -r store`。 ### Windows @@ -73,7 +73,7 @@ UploadPartCopy,AbortMultipartUpload,ListMultipartUpload,ListParts - 下载 [oss-emulator](https://github.com/aliyun/oss-emulator) -- 运行。进入 *oss-emulator* 目录, 执行命令 `ruby bin/emulator -r store -p 8080`。 +- 运行。进入 *oss-emulator* 目录, 执行命令 `ruby bin/emulator -r store`。 ## 使用示例 @@ -81,7 +81,7 @@ UploadPartCopy,AbortMultipartUpload,ListMultipartUpload,ListParts - 方法一：直接在命令行中携带参数, 其中endpoint设置为oss-emulator的IP; AccessKeyId和AccessKeySecret如下, 也可以不填。 如： ``` - ossutil -e http://192.168.0.1:8080 -i AccessKeyId -k AccessKeySecret ls oss://bucket + ossutil -e http://192.168.0.1 -i AccessKeyId -k AccessKeySecret ls oss://bucket ``` - 方法二：使用 `ossutil config` 命令配置参数，参数配置和 **方法一** 相同： @@ -100,7 +100,7 @@ UploadPartCopy,AbortMultipartUpload,ListMultipartUpload,ListParts import oss2 auth = oss2.Auth('AccessKeySecret', 'AccessKeySecret') - bucket = oss2.Bucket(auth, 'http//:192.168.0.1:8080', 'MyBucketName') + bucket = oss2.Bucket(auth, 'http://192.168.0.1', 'MyBucketName') bucket.create_bucket() ``` diff --git a/lib/emulator/cmdline.rb b/lib/emulator/cmdline.rb index 55e2a04..cc4a656 100644 --- a/lib/emulator/cmdline.rb +++ b/lib/emulator/cmdline.rb @@ -20,6 +20,10 @@ def server Config.init() Config.set_quiet_mode(options[:quiet]) Config.set_log_level(options[:loglevel].downcase) if options[:loglevel] + + if RUBY_VERSION < '2.2.8' && RUBY_ENGINE == 'ruby' + abort("The Ruby version should be above 2.2.8 , current Ruby version is #{RUBY_VERSION} . ") + end if options[:root] root_dir = File.expand_path(options[:root]) diff --git a/lib/emulator/logging.rb b/lib/emulator/logging.rb index 3949071..5b1c115 100644 --- a/lib/emulator/logging.rb +++ b/lib/emulator/logging.rb @@ -14,6 +14,7 @@ module OssEmulator # Log.set_quiet_mode(false) # Log.info("something", 'green') # Log.fatal("something", 'red') + # 打印调试信息 ruby emulator -r ../store -q false -L debug class Log LOG_DEFAULT_DIR = "../log" diff --git a/lib/emulator/multipart.rb b/lib/emulator/multipart.rb index 3cb0d81..981ecb4 100644 --- a/lib/emulator/multipart.rb +++ b/lib/emulator/multipart.rb @@ -3,6 +3,7 @@ require "rexml/document" require 'emulator/config' require 'emulator/util' +require 'emulator/request' require 'emulator/response' include REXML include Comparable @@ -16,7 +17,7 @@ def self.initiate_multipart_upload(bucket, object, response) return if OssResponse.response_no_such_bucket(response, bucket) # delete object - OssUtil.delete_object_file_and_dir(bucket, object) + #OssUtil.delete_object_file_and_dir(bucket, object) dataset = { cmd: Request::POST_INIT_MULTIPART_UPLOAD, @@ -30,9 +31,10 @@ def self.initiate_multipart_upload(bucket, object, response) # UploadPart def self.upload_part(req, query, request, response) - part_number = query['partNumber'].first + part_number = query['partNumber'].first + uploadId = query['uploadId'].first - Object.put_object(req.bucket, req.object, request, response, part_number) + Object.put_object(req.bucket, req.object, request, response, part_number, uploadId) end # CompleteMultipartUpload @@ -48,14 +50,16 @@ def self.complete_multipart_upload(req, request, response) end object_dir = File.join(Config.store, req.bucket, req.object) - base_obj_part_filename = File.join(object_dir, Store::OBJECT_CONTENT_PREFIX) + temp_subdir = req.query_parser['uploadId'].first + temp_object_dir = File.join(object_dir, temp_subdir) + base_obj_part_filename = File.join(temp_object_dir, Store::OBJECT_CONTENT_PREFIX) complete_file_size = 0 parts.each do |part| part_filename = "#{base_obj_part_filename}#{part[:number]}" complete_file_size += File.size(part_filename) end - options = { :size => complete_file_size, :part_size => File.size(File.join(object_dir, Store::OBJECT_CONTENT)) } + options = { :temp_dir => temp_subdir, :size => complete_file_size, :part_size => File.size(File.join(temp_object_dir, Store::OBJECT_CONTENT)) } dataset = OssUtil.put_object_metadata(req.bucket, req.object, request, options) dataset[:cmd] = Request::POST_COMPLETE_MULTIPART_UPLOAD diff --git a/lib/emulator/object.rb b/lib/emulator/object.rb index 23bbc36..6d8cab0 100644 --- a/lib/emulator/object.rb +++ b/lib/emulator/object.rb @@ -10,7 +10,7 @@ module OssEmulator module Object # PutObject - def self.put_object(bucket, object, request, response, part_number=nil) + def self.put_object(bucket, object, request, response, part_number=nil, uploadId=nil) # NoSuchBucket return if OssResponse.response_no_such_bucket(response, bucket) @@ -44,13 +44,22 @@ def self.put_object(bucket, object, request, response, part_number=nil) end obj_dir = File.join(Config.store, bucket, object) + temp_subdir = "" + temp_obj_dir = "" + Log.debug("request.header['authorization']=#{request.header['authorization']}") if part_number - object_content_filename = File.join(obj_dir, "#{Store::OBJECT_CONTENT_PREFIX}#{part_number}") + temp_subdir = uploadId + Log.debug("temp_subdir=#{temp_subdir}", 'green') + temp_obj_dir = File.join(obj_dir, temp_subdir) + object_content_filename = File.join(temp_obj_dir, "#{Store::OBJECT_CONTENT_PREFIX}#{part_number}") else - OssUtil.delete_object_file_and_dir(bucket, object) - object_content_filename = File.join(obj_dir, Store::OBJECT_CONTENT) + #OssUtil.delete_object_file_and_dir(bucket, object) + temp_subdir = request.header['authorization'].first.split(':')[1].gsub(/[^a-zA-Z0-9]/, '') + Log.debug("temp_subdir=#{temp_subdir}", 'green') + temp_obj_dir = File.join(obj_dir, temp_subdir) + object_content_filename = File.join(temp_obj_dir, Store::OBJECT_CONTENT) end - FileUtils.mkdir_p(obj_dir) unless File.exist?(obj_dir) + FileUtils.mkdir_p(temp_obj_dir) unless File.exist?(temp_obj_dir) f_object_content = File.new(object_content_filename, 'a') f_object_content.binmode @@ -83,7 +92,8 @@ def self.put_object(bucket, object, request, response, part_number=nil) dataset = {} # put object metadata if not multipart upload - dataset = OssUtil.put_object_metadata(bucket, object, request) unless part_number + option = { temp_dir: temp_subdir } + dataset = OssUtil.put_object_metadata(bucket, object, request, option) unless part_number dataset[:cmd] = Request::PUT_OBJECT OssResponse.response_ok(response, dataset) @@ -94,6 +104,8 @@ def self.copy_object(src_bucket, src_object, dst_bucket, dst_object, request, re src_object_dir = File.join(Config.store, src_bucket, src_object) src_metadata_filename = File.join(src_object_dir, Store::OBJECT_METADATA) + #temp_subdir = request.header['authorization'].first.split(':')[1].gsub(/[^a-zA-Z0-9]/, '') + #dst_object_dir = File.join(Config.store, dst_bucket, dst_object, temp_subdir) dst_object_dir = File.join(Config.store, dst_bucket, dst_object) dst_metadata_filename = File.join(dst_object_dir, Store::OBJECT_METADATA) diff --git a/lib/emulator/util.rb b/lib/emulator/util.rb index 9f8c021..cecc783 100644 --- a/lib/emulator/util.rb +++ b/lib/emulator/util.rb @@ -3,6 +3,7 @@ require 'time' require 'fileutils' require 'builder' +require "thread" require 'emulator/config' module OssEmulator @@ -159,16 +160,20 @@ def self.get_bucket_list_objects(lbr, req) end #get_bucket_list_objects def self.delete_object_file_and_dir(bucket, object) + Log.debug("delete_object_file_and_dir", 'yellow') bucket_dir = File.join(Config.store, bucket) object_dir = File.join(bucket_dir, object) object_metadata_filename = File.join(object_dir, Store::OBJECT_METADATA) FileUtils.rm_rf(object_metadata_filename) if File.exist?(object_metadata_filename) + object_content_filename = File.join(object_dir, Store::OBJECT_CONTENT_PREFIX) + FileUtils.rm_rf("#{object_content_filename}*") if File.exist?(object_content_filename) current_level_folder = object_dir while File.exist?(current_level_folder) return if current_level_folder==bucket_dir Find.find(current_level_folder) do |filename| + Log.debug("filename", 'blue') return if filename.include?(Store::OBJECT_METADATA) end @@ -179,8 +184,18 @@ def self.delete_object_file_and_dir(bucket, object) def self.put_object_metadata(bucket, object, request, options = nil) obj_dir = File.join(Config.store, bucket, object) - content_filename = File.join(obj_dir, Store::OBJECT_CONTENT) - Log.raise("put_object_metadata : Object content file does not exist in put_object_metadata : #{obj_dir}") unless File.exist?(content_filename) + + temp_obj_dir = obj_dir + temp_subdir = '' + if options!=nil && options.include?(:temp_dir) + temp_subdir = options[:temp_dir] if options.include?(:temp_dir) + temp_obj_dir = File.join(obj_dir, temp_subdir) + Log.debug("obj_dir=#{obj_dir}, options=#{options}, temp_subdir=#{temp_subdir}") + end + + content_filename = File.join(temp_obj_dir, Store::OBJECT_CONTENT) + + Log.raise("put_object_metadata : Object content file does not exist in put_object_metadata : #{temp_obj_dir}") unless File.exist?(content_filename) # construct metadata metadata = {} @@ -220,11 +235,44 @@ def self.put_object_metadata(bucket, object, request, options = nil) end # store metadata to file - metadata_file = File.join(obj_dir, Store::OBJECT_METADATA) + metadata_file = File.join(temp_obj_dir, Store::OBJECT_METADATA) File.open(metadata_file, 'w') do |f| f << YAML::dump(metadata) end + # define the dynamic metux var + if temp_subdir!='' + # bucket, object, str_var_metux = "@mutex_#{temp_subdir}" + bucket_var = bucket.gsub(/\W/, '_') + object_var = object.gsub(/\W/, '_') + str_var_metux = "@mutex_#{bucket_var}_#{object_var}" + instance_variable_set(str_var_metux, Mutex.new) + Log.debug(instance_variable_get(str_var_metux), 'blue') + # metux lock + instance_variable_get(str_var_metux).lock + + # remove old object files + list_oldfiles = Dir.entries(obj_dir) + list_oldfiles.each do |f| + if f.include?("_object_oss_aliyun_ALIBABA") + filepath = "#{obj_dir}/#{f}" + Log.debug(filepath, "yellow") + FileUtils.rm_rf(filepath) + end + end + + # move object files from temp_dir to normal dir + Log.info("move temp_object to normal object. ") + list_move = Dir.entries(temp_obj_dir) + list_move.each do |f| + FileUtils.mv "#{temp_obj_dir}/#{f}",obj_dir if !File.directory?(f) + end + FileUtils.rm_rf(temp_obj_dir) if File.exist?(temp_obj_dir) + + # metux unlock + instance_variable_get(str_var_metux).unlock + end + metadata end From 67eb970cd053112fba06359bac531f39ba83595c Mon Sep 17 00:00:00 2001 From: wb-lhx292136 Date: Sun, 8 Apr 2018 21:14:09 +0800 Subject: [PATCH 2/6] remove comments in copy_object --- lib/emulator/object.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/emulator/object.rb b/lib/emulator/object.rb index 6d8cab0..5dcff9c 100644 --- a/lib/emulator/object.rb +++ b/lib/emulator/object.rb @@ -104,8 +104,6 @@ def self.copy_object(src_bucket, src_object, dst_bucket, dst_object, request, re src_object_dir = File.join(Config.store, src_bucket, src_object) src_metadata_filename = File.join(src_object_dir, Store::OBJECT_METADATA) - #temp_subdir = request.header['authorization'].first.split(':')[1].gsub(/[^a-zA-Z0-9]/, '') - #dst_object_dir = File.join(Config.store, dst_bucket, dst_object, temp_subdir) dst_object_dir = File.join(Config.store, dst_bucket, dst_object) dst_metadata_filename = File.join(dst_object_dir, Store::OBJECT_METADATA) From 3ebd52f2aa3c74ddf1b01199b89be961d22392d1 Mon Sep 17 00:00:00 2001 From: wb-lhx292136 Date: Mon, 9 Apr 2018 09:14:28 +0800 Subject: [PATCH 3/6] remove comments for delete part in initiate_multipart_upload --- lib/emulator/multipart.rb | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/emulator/multipart.rb b/lib/emulator/multipart.rb index 981ecb4..b3d6596 100644 --- a/lib/emulator/multipart.rb +++ b/lib/emulator/multipart.rb @@ -16,9 +16,6 @@ def self.initiate_multipart_upload(bucket, object, response) # NoSuchBucket return if OssResponse.response_no_such_bucket(response, bucket) - # delete object - #OssUtil.delete_object_file_and_dir(bucket, object) - dataset = { cmd: Request::POST_INIT_MULTIPART_UPLOAD, bucket: bucket, From c507ac26e1f10a9d58a117cb3c03d4337a23cfcb Mon Sep 17 00:00:00 2001 From: wb-lhx292136 Date: Mon, 16 Apr 2018 10:19:04 +0800 Subject: [PATCH 4/6] update mutex handling. --- lib/emulator/util.rb | 54 ++++++++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/lib/emulator/util.rb b/lib/emulator/util.rb index cecc783..06daed8 100644 --- a/lib/emulator/util.rb +++ b/lib/emulator/util.rb @@ -240,37 +240,41 @@ def self.put_object_metadata(bucket, object, request, options = nil) f << YAML::dump(metadata) end - # define the dynamic metux var + # define the dynamic mutex var if temp_subdir!='' - # bucket, object, str_var_metux = "@mutex_#{temp_subdir}" + # bucket, object, str_var_mutex = "@mutex_#{temp_subdir}" bucket_var = bucket.gsub(/\W/, '_') object_var = object.gsub(/\W/, '_') - str_var_metux = "@mutex_#{bucket_var}_#{object_var}" - instance_variable_set(str_var_metux, Mutex.new) - Log.debug(instance_variable_get(str_var_metux), 'blue') - # metux lock - instance_variable_get(str_var_metux).lock - - # remove old object files - list_oldfiles = Dir.entries(obj_dir) - list_oldfiles.each do |f| - if f.include?("_object_oss_aliyun_ALIBABA") - filepath = "#{obj_dir}/#{f}" - Log.debug(filepath, "yellow") - FileUtils.rm_rf(filepath) + str_var_mutex = "@mutex_#{bucket_var}_#{object_var}" + instance_variable_set(str_var_mutex, Mutex.new) + Log.debug(instance_variable_get(str_var_mutex), 'blue') + + begin + # mutex lock + instance_variable_get(str_var_mutex).lock + + # remove old object files + list_oldfiles = Dir.entries(obj_dir) + list_oldfiles.each do |f| + if f.include?("_object_oss_aliyun_ALIBABA") + filepath = "#{obj_dir}/#{f}" + Log.debug(filepath, "yellow") + FileUtils.rm_rf(filepath) + end end - end - # move object files from temp_dir to normal dir - Log.info("move temp_object to normal object. ") - list_move = Dir.entries(temp_obj_dir) - list_move.each do |f| - FileUtils.mv "#{temp_obj_dir}/#{f}",obj_dir if !File.directory?(f) - end - FileUtils.rm_rf(temp_obj_dir) if File.exist?(temp_obj_dir) + # move object files from temp_dir to normal dir + Log.info("move temp_object to normal object. ") + list_move = Dir.entries(temp_obj_dir) + list_move.each do |f| + FileUtils.mv "#{temp_obj_dir}/#{f}",obj_dir if !File.directory?(f) + end + FileUtils.rm_rf(temp_obj_dir) if File.exist?(temp_obj_dir) - # metux unlock - instance_variable_get(str_var_metux).unlock + ensure + # mutex unlock + instance_variable_get(str_var_mutex).unlock + end end metadata From 83be974fbd7f2b9597943c99f546fbb4ae4a915c Mon Sep 17 00:00:00 2001 From: wb-lhx292136 Date: Mon, 16 Apr 2018 14:03:03 +0800 Subject: [PATCH 5/6] change mutex.lock to synchronize mode. --- lib/emulator/util.rb | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/lib/emulator/util.rb b/lib/emulator/util.rb index 06daed8..799f20d 100644 --- a/lib/emulator/util.rb +++ b/lib/emulator/util.rb @@ -246,14 +246,14 @@ def self.put_object_metadata(bucket, object, request, options = nil) bucket_var = bucket.gsub(/\W/, '_') object_var = object.gsub(/\W/, '_') str_var_mutex = "@mutex_#{bucket_var}_#{object_var}" - instance_variable_set(str_var_mutex, Mutex.new) - Log.debug(instance_variable_get(str_var_mutex), 'blue') + mutex_set = instance_variable_set(str_var_mutex, Mutex.new) + thread_id = Thread.current + mutex_status = instance_variable_get(str_var_mutex).locked? + Log.debug("#{thread_id} :: Generate Mutex=#{mutex_set},status=#{mutex_status}; #{str_var_mutex}", 'blue') - begin - # mutex lock - instance_variable_get(str_var_mutex).lock - + instance_variable_get(str_var_mutex).synchronize{ # remove old object files + Log.debug("#{thread_id} :: #{str_var_mutex} : enter synchronize block and remove old object. ", 'blue') list_oldfiles = Dir.entries(obj_dir) list_oldfiles.each do |f| if f.include?("_object_oss_aliyun_ALIBABA") @@ -264,17 +264,14 @@ def self.put_object_metadata(bucket, object, request, options = nil) end # move object files from temp_dir to normal dir - Log.info("move temp_object to normal object. ") + Log.info("#{thread_id} :: #{str_var_mutex} : move temp_object to normal object. ", 'blue') list_move = Dir.entries(temp_obj_dir) list_move.each do |f| FileUtils.mv "#{temp_obj_dir}/#{f}",obj_dir if !File.directory?(f) end FileUtils.rm_rf(temp_obj_dir) if File.exist?(temp_obj_dir) - - ensure - # mutex unlock - instance_variable_get(str_var_mutex).unlock - end + Log.debug("#{thread_id} :: #{str_var_mutex} : quit synchronize block. ", 'blue') + } end metadata From a9ec88ab58d584311b07f0763da369f90be53371 Mon Sep 17 00:00:00 2001 From: wb-lhx292136 Date: Tue, 8 May 2018 16:18:35 +0800 Subject: [PATCH 6/6] udpate README.md for default port . --- README.md | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 90df667..a1ed97c 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ UploadPartCopy,AbortMultipartUpload,ListMultipartUpload,ListParts - 下载 [oss-emulator](https://github.com/aliyun/oss-emulator) -- 运行。进入 *oss-emulator* 目录, 执行命令 `ruby bin/emulator -r store`。 +- 运行。进入 *oss-emulator* 目录, 执行命令 `ruby bin/emulator -r store`, 默认监听端口为80; 如果要指定其他端口(比如8080), 则执行命令`ruby bin/emulator -r store -p 8080`。 ### Windows @@ -73,16 +73,20 @@ UploadPartCopy,AbortMultipartUpload,ListMultipartUpload,ListParts - 下载 [oss-emulator](https://github.com/aliyun/oss-emulator) -- 运行。进入 *oss-emulator* 目录, 执行命令 `ruby bin/emulator -r store`。 +- 运行。进入 *oss-emulator* 目录, 执行命令 `ruby bin/emulator -r store`, 默认监听端口为80; 如果要指定其他端口(比如8080), 则执行命令`ruby bin/emulator -r store -p 8080`。 ## 使用示例 ### ossutil -- 方法一：直接在命令行中携带参数, 其中endpoint设置为oss-emulator的IP; AccessKeyId和AccessKeySecret如下, 也可以不填。 如： +- 方法一：直接在命令行中携带参数, 其中endpoint设置为oss-emulator的IP; AccessKeyId和AccessKeySecret如下, 也可以不填; 默认连接到服务器端口80。 如： ``` ossutil -e http://192.168.0.1 -i AccessKeyId -k AccessKeySecret ls oss://bucket ``` + 如果要指定连接到其他服务器端口(比如8080), 则按照如下命令来执行： +``` + ossutil -e http://192.168.0.1:8080 -i AccessKeyId -k AccessKeySecret ls oss://bucket +``` - 方法二：使用 `ossutil config` 命令配置参数，参数配置和 **方法一** 相同： ``` @@ -94,7 +98,7 @@ UploadPartCopy,AbortMultipartUpload,ListMultipartUpload,ListParts ### Python SDK -- *Python SDK* 连接 oss-emulator 代码的如下, 其中endpoint设置为 oss-emulator 的IP, AccessKeyId和AccessKeySecret如下, 也可以不填。 +- *Python SDK* 连接 oss-emulator 代码的如下, 其中endpoint设置为 oss-emulator 的IP, AccessKeyId和AccessKeySecret如下, 也可以不填; 默认连接到服务器端口80。 ``` import oss2 @@ -103,6 +107,14 @@ UploadPartCopy,AbortMultipartUpload,ListMultipartUpload,ListParts bucket = oss2.Bucket(auth, 'http://192.168.0.1', 'MyBucketName') bucket.create_bucket() ``` +- 如果要指定连接到其他服务器端口(比如8080), 则按照如下代码来编写： +``` + import oss2 + + auth = oss2.Auth('AccessKeySecret', 'AccessKeySecret') + bucket = oss2.Bucket(auth, 'http//:192.168.0.1:8080', 'MyBucketName') + bucket.create_bucket() +``` > **提示：** - Python SDK的说明文档请参考[官网](https://help.aliyun.com/document_detail/32026.html?spm=5176.doc32026.3.3.RQzyY1) \ No newline at end of file