diff options
author | Yinhang Liu <yinhangx.liu@intel.com> | 2017-08-24 09:52:52 +0800 |
---|---|---|
committer | windyuan <feng.yuan@intel.com> | 2017-08-28 18:03:35 +0800 |
commit | 77853bb1f742485447495bf7f384a8abe8669c52 (patch) | |
tree | 004878cd574d37e7039d02337c5d6bde767b0cb8 | |
parent | 0995e53b9064a563b343a9bdcd50ecf88bf84bec (diff) | |
download | libxcam-77853bb1f742485447495bf7f384a8abe8669c52.tar.gz |
cl-stitch: support multiple cameras
* add 4 cameras mode
* optimize feature match to support single match at a time
* prepare parameters for each blender independently
* swap buffer index for the last blender in configuration phase
* test-image-stitching cmdline:
$ test-image-stitching --input input0.nv12 --input input1.nv12 \
--input input2.nv12 --input input3.nv12 --output output.mp4 \
--input-w 1280 --input-h 800 --output-w 4864 --output-h 1408 \
--scale-mode local --enable-fisheyemap --res-mode 1080p4 \
--fm-ocl false --framerate 30.0 --save true \
--fisheye-num 4 --all-in-one false
-rw-r--r-- | modules/interface/stitcher.h | 19 | ||||
-rw-r--r-- | modules/ocl/cl_image_360_stitch.cpp | 560 | ||||
-rw-r--r-- | modules/ocl/cl_image_360_stitch.h | 38 | ||||
-rw-r--r-- | modules/ocl/cl_pyramid_blender.cpp | 2 | ||||
-rw-r--r-- | modules/ocl/cv_feature_match.cpp | 147 | ||||
-rw-r--r-- | modules/ocl/cv_feature_match.h | 24 | ||||
-rw-r--r-- | tests/test-image-stitching.cpp | 208 | ||||
-rw-r--r-- | wrapper/gstreamer/gstxcamfilter.h | 2 |
8 files changed, 550 insertions, 450 deletions
diff --git a/modules/interface/stitcher.h b/modules/interface/stitcher.h index 91aef71..31a6f8c 100644 --- a/modules/interface/stitcher.h +++ b/modules/interface/stitcher.h @@ -26,24 +26,21 @@ #include "interface/data_types.h" #include "video_buffer.h" +#define XCAM_STITCH_FISHEYE_MAX_NUM 6 + namespace XCam { enum StitchResMode { StitchRes1080P, + StitchRes1080P4, StitchRes4K }; -enum ImageIdx { - ImageIdxMain, - ImageIdxSecondary, - ImageIdxCount, -}; - struct StitchInfo { - uint32_t merge_width[ImageIdxCount]; + uint32_t merge_width[XCAM_STITCH_FISHEYE_MAX_NUM]; - ImageCropInfo crop[ImageIdxCount]; - FisheyeInfo fisheye_info[ImageIdxCount]; + ImageCropInfo crop[XCAM_STITCH_FISHEYE_MAX_NUM]; + FisheyeInfo fisheye_info[XCAM_STITCH_FISHEYE_MAX_NUM]; StitchInfo () { xcam_mem_clear (merge_width); @@ -81,8 +78,8 @@ private: uint32_t _output_width; uint32_t _output_height; - ImageMergeInfo _img_merge_info[ImageIdxCount]; - Rect _overlaps[ImageIdxCount][2]; // 2=>Overlap0 and overlap1 + ImageMergeInfo _img_merge_info[XCAM_STITCH_FISHEYE_MAX_NUM]; + Rect _overlaps[XCAM_STITCH_FISHEYE_MAX_NUM][2]; // 2=>Overlap0 and overlap1 StitchInfo _stitch_info; bool _is_stitch_inited; diff --git a/modules/ocl/cl_image_360_stitch.cpp b/modules/ocl/cl_image_360_stitch.cpp index ff2d338..954a8ed 100644 --- a/modules/ocl/cl_image_360_stitch.cpp +++ b/modules/ocl/cl_image_360_stitch.cpp @@ -120,6 +120,15 @@ get_fm_default_config (StitchResMode res_mode) break; } + case StitchRes1080P4: { + config.sitch_min_width = 96; + config.min_corners = 8; + config.offset_factor = 0.8f; + config.delta_mean_offset = 5.0f; + config.max_adjusted_offset = 12.0f; + + break; + } case StitchRes4K: { config.sitch_min_width = 160; config.min_corners = 8; @@ -167,7 +176,51 @@ get_default_stitch_info (StitchResMode res_mode) stitch_info.fisheye_info[1].wide_angle = 202.8f; stitch_info.fisheye_info[1].radius = 480.0f; stitch_info.fisheye_info[1].rotate_angle = 89.4f; - + break; + } + case StitchRes1080P4: { + stitch_info.merge_width[0] = 96; + stitch_info.merge_width[1] = 96; + stitch_info.merge_width[2] = 96; + stitch_info.merge_width[3] = 96; + + stitch_info.crop[0].left = 16; + stitch_info.crop[0].right = 16; + stitch_info.crop[0].top = 0; + stitch_info.crop[0].bottom = 0; + stitch_info.crop[1].left = 16; + stitch_info.crop[1].right = 16; + stitch_info.crop[1].top = 0; + stitch_info.crop[1].bottom = 0; + stitch_info.crop[2].left = 16; + stitch_info.crop[2].right = 16; + stitch_info.crop[2].top = 0; + stitch_info.crop[2].bottom = 0; + stitch_info.crop[3].left = 16; + stitch_info.crop[3].right = 16; + stitch_info.crop[3].top = 0; + stitch_info.crop[3].bottom = 0; + + stitch_info.fisheye_info[0].center_x = 640.0f; + stitch_info.fisheye_info[0].center_y = 400.0f; + stitch_info.fisheye_info[0].wide_angle = 120.0f; + stitch_info.fisheye_info[0].radius = 640.0f; + stitch_info.fisheye_info[0].rotate_angle = 0.0f; + stitch_info.fisheye_info[1].center_x = 640.0f; + stitch_info.fisheye_info[1].center_y = 400.0f; + stitch_info.fisheye_info[1].wide_angle = 120.0f; + stitch_info.fisheye_info[1].radius = 640.0f; + stitch_info.fisheye_info[1].rotate_angle = 0.0f; + stitch_info.fisheye_info[2].center_x = 640.0f; + stitch_info.fisheye_info[2].center_y = 400.0f; + stitch_info.fisheye_info[2].wide_angle = 120.0f; + stitch_info.fisheye_info[2].radius = 640.0f; + stitch_info.fisheye_info[2].rotate_angle = 0.0f; + stitch_info.fisheye_info[3].center_x = 640.0f; + stitch_info.fisheye_info[3].center_y = 400.0f; + stitch_info.fisheye_info[3].wide_angle = 120.0f; + stitch_info.fisheye_info[3].radius = 640.0f; + stitch_info.fisheye_info[3].rotate_angle = 0.0f; break; } case StitchRes4K: { @@ -194,7 +247,6 @@ get_default_stitch_info (StitchResMode res_mode) stitch_info.fisheye_info[1].wide_angle = 192.0f; stitch_info.fisheye_info[1].radius = 1040.0f; stitch_info.fisheye_info[1].rotate_angle = 0.4f; - break; } default: @@ -206,7 +258,8 @@ get_default_stitch_info (StitchResMode res_mode) } CLImage360Stitch::CLImage360Stitch ( - const SmartPtr<CLContext> &context, CLBlenderScaleMode scale_mode, StitchResMode res_mode) + const SmartPtr<CLContext> &context, CLBlenderScaleMode scale_mode, StitchResMode res_mode, + int fisheye_num, bool all_in_one_img) : CLMultiImageHandler (context, "CLImage360Stitch") , _context (context) , _output_width (0) @@ -214,12 +267,16 @@ CLImage360Stitch::CLImage360Stitch ( , _scale_mode (scale_mode) , _res_mode (res_mode) , _is_stitch_inited (false) + , _fisheye_num (fisheye_num) + , _all_in_one_img (all_in_one_img) { #if HAVE_OPENCV - _feature_match = new CVFeatureMatch (); - XCAM_ASSERT (_feature_match.ptr ()); - - _feature_match->set_config (get_fm_default_config (res_mode)); + for (int i = 0; i < fisheye_num; i++) { + _feature_match[i] = new CVFeatureMatch (); + XCAM_ASSERT (_feature_match[i].ptr ()); + _feature_match[i]->set_config (get_fm_default_config (res_mode)); + _feature_match[i]->set_fm_index (i); + } #endif } @@ -231,7 +288,7 @@ CLImage360Stitch::set_stitch_info (StitchInfo stitch_info) return false; } - for (int index = 0; index < ImageIdxCount; ++index) { + for (int index = 0; index < _fisheye_num; ++index) { _fisheye[index].handler->set_fisheye_info (stitch_info.fisheye_info[index]); } @@ -255,7 +312,7 @@ CLImage360Stitch::get_stitch_info () bool CLImage360Stitch::set_fisheye_handler (SmartPtr<CLFisheyeHandler> fisheye, int index) { - XCAM_ASSERT (index < ImageIdxCount); + XCAM_ASSERT (index < _fisheye_num); _fisheye[index].handler = fisheye; SmartPtr<CLImageHandler> handler = fisheye; @@ -263,18 +320,9 @@ CLImage360Stitch::set_fisheye_handler (SmartPtr<CLFisheyeHandler> fisheye, int i } bool -CLImage360Stitch::set_left_blender (SmartPtr<CLBlender> blender) -{ - _left_blender = blender; - - SmartPtr<CLImageHandler> handler = blender; - return add_image_handler (handler); -} - -bool -CLImage360Stitch::set_right_blender (SmartPtr<CLBlender> blender) +CLImage360Stitch::set_blender (SmartPtr<CLBlender> blender, int idx) { - _right_blender = blender; + _blender[idx] = blender; SmartPtr<CLImageHandler> handler = blender; return add_image_handler (handler); @@ -283,7 +331,7 @@ CLImage360Stitch::set_right_blender (SmartPtr<CLBlender> blender) bool CLImage360Stitch::set_image_overlap (const int idx, const Rect &overlap0, const Rect &overlap1) { - XCAM_ASSERT (idx < ImageIdxCount); + XCAM_ASSERT (idx < _fisheye_num); _overlaps[idx][0] = overlap0; _overlaps[idx][1] = overlap1; return true; @@ -293,7 +341,9 @@ void CLImage360Stitch::set_feature_match_ocl (bool fm_ocl) { #if HAVE_OPENCV - _feature_match->set_ocl (fm_ocl); + for (int i = 0; i < _fisheye_num; i++) { + _feature_match[i]->set_ocl (fm_ocl); + } #else XCAM_UNUSED (fm_ocl); XCAM_LOG_WARNING ("non-OpenCV mode, failed to set ocl for feature match"); @@ -302,15 +352,15 @@ CLImage360Stitch::set_feature_match_ocl (bool fm_ocl) #if HAVE_OPENCV void -CLImage360Stitch::set_feature_match_config (CVFMConfig config) +CLImage360Stitch::set_feature_match_config (const int idx, CVFMConfig config) { - _feature_match->set_config (config); + _feature_match[idx]->set_config (config); } CVFMConfig -CLImage360Stitch::get_feature_match_config () +CLImage360Stitch::get_feature_match_config (const int idx) { - return _feature_match->get_config (); + return _feature_match[idx]->get_config (); } #endif @@ -318,22 +368,31 @@ void CLImage360Stitch::calc_fisheye_initial_info (SmartPtr<DrmBoBuffer> &output) { const VideoBufferInfo &out_info = output->get_video_info (); - _fisheye[0].width = (out_info.width + _stitch_info.merge_width[0] + _stitch_info.merge_width[1] - + _stitch_info.crop[0].left + _stitch_info.crop[0].right - + _stitch_info.crop[1].left + _stitch_info.crop[1].right) / 2; + + uint32_t fisheye_width_sum = out_info.width; + for (int i = 0; i < _fisheye_num; i++) { + fisheye_width_sum += _stitch_info.merge_width[i] + _stitch_info.crop[i].left + _stitch_info.crop[i].right; + } + _fisheye[0].width = fisheye_width_sum / _fisheye_num; _fisheye[0].width = XCAM_ALIGN_UP (_fisheye[0].width, 16); _fisheye[0].height = out_info.height + _stitch_info.crop[0].top + _stitch_info.crop[0].bottom; XCAM_LOG_INFO ( "fisheye correction output size width:%d height:%d", _fisheye[0].width, _fisheye[0].height); - _fisheye[1].width = _fisheye[0].width; - _fisheye[1].height = _fisheye[0].height; + for (int i = 1; i < _fisheye_num; i++) { + _fisheye[i].width = _fisheye[0].width; + _fisheye[i].height = _fisheye[0].height; + } + + float max_dst_longitude, max_dst_latitude; + for (int i = 0; i < _fisheye_num; ++i) { + max_dst_latitude = (_stitch_info.fisheye_info[i].wide_angle > 180.0f) ? + 180.0f : _stitch_info.fisheye_info[i].wide_angle; + max_dst_longitude = max_dst_latitude * _fisheye[i].width / _fisheye[i].height; - float max_dst_angle = 180.0f * _fisheye[0].width / _fisheye[0].height; - for (int index = 0; index < ImageIdxCount; ++index) { - _fisheye[index].handler->set_dst_range (max_dst_angle, 180.0f); - _fisheye[index].handler->set_output_size (_fisheye[index].width, _fisheye[index].height); + _fisheye[i].handler->set_dst_range (max_dst_longitude, max_dst_latitude); + _fisheye[i].handler->set_output_size (_fisheye[i].width, _fisheye[i].height); } } @@ -342,30 +401,30 @@ CLImage360Stitch::update_image_overlap () { static bool is_merge_info_inited = false; if (!is_merge_info_inited) { - _img_merge_info[0].left.pos_x = _stitch_info.crop[0].left; - _img_merge_info[0].left.pos_y = _stitch_info.crop[0].top; - _img_merge_info[0].left.width = _stitch_info.merge_width[0]; - _img_merge_info[0].left.height = _fisheye[0].height - _stitch_info.crop[0].top - _stitch_info.crop[0].bottom; - - _img_merge_info[0].right.pos_x = _fisheye[0].width - _stitch_info.crop[0].right - _stitch_info.merge_width[1]; - _img_merge_info[0].right.pos_y = _stitch_info.crop[0].top; - _img_merge_info[0].right.width = _stitch_info.merge_width[1]; - _img_merge_info[0].right.height = _fisheye[0].height - _stitch_info.crop[0].top - _stitch_info.crop[0].bottom; - - _img_merge_info[1].left.pos_x = _stitch_info.crop[1].left; - _img_merge_info[1].left.pos_y = _stitch_info.crop[1].top; - _img_merge_info[1].left.width = _stitch_info.merge_width[1]; - _img_merge_info[1].left.height = _fisheye[1].height - _stitch_info.crop[1].top - _stitch_info.crop[1].bottom; - _img_merge_info[1].right.pos_x = _fisheye[1].width - _stitch_info.crop[1].right - _stitch_info.merge_width[0]; - _img_merge_info[1].right.pos_y = _stitch_info.crop[1].top; - _img_merge_info[1].right.width = _stitch_info.merge_width[0]; - _img_merge_info[1].right.height = _fisheye[0].height - _stitch_info.crop[1].top - _stitch_info.crop[1].bottom; + int idx_next = 1; + for (int i = 0; i < _fisheye_num; i++) { + idx_next = (i == (_fisheye_num - 1)) ? 0 : (i + 1); + + _img_merge_info[i].left.pos_x = _stitch_info.crop[i].left; + _img_merge_info[i].left.pos_y = _stitch_info.crop[i].top; + _img_merge_info[i].left.width = _stitch_info.merge_width[i]; + _img_merge_info[i].left.height = _fisheye[i].height - _stitch_info.crop[i].top + - _stitch_info.crop[i].bottom; + + _img_merge_info[i].right.pos_x = _fisheye[i].width - _stitch_info.crop[i].right + - _stitch_info.merge_width[idx_next]; + _img_merge_info[i].right.pos_y = _stitch_info.crop[i].top; + _img_merge_info[i].right.width = _stitch_info.merge_width[idx_next]; + _img_merge_info[i].right.height = _fisheye[i].height - _stitch_info.crop[i].top + - _stitch_info.crop[i].bottom; + } is_merge_info_inited = true; } - set_image_overlap (0, _img_merge_info[0].left, _img_merge_info[0].right); - set_image_overlap (1, _img_merge_info[1].left, _img_merge_info[1].right); + for (int i = 0; i < _fisheye_num; i++) { + set_image_overlap (i, _img_merge_info[i].left, _img_merge_info[i].right); + } } XCamReturn @@ -373,15 +432,9 @@ CLImage360Stitch::prepare_buffer_pool_video_info ( const VideoBufferInfo &input, VideoBufferInfo &output) { if (_output_width == 0 || _output_height == 0) { - _output_width = input.width; - _output_height = XCAM_ALIGN_UP (input.width / 2, 16); + XCAM_LOG_ERROR ("incorrect output size: width:%d height:%d", _output_width, _output_height); + return XCAM_RETURN_ERROR_PARAM; } - XCAM_FAIL_RETURN( - WARNING, - _output_width && _output_height && (_output_width == _output_height * 2), - XCAM_RETURN_ERROR_PARAM, - "CLImage360Stitch(%s) prepare buffer pool info failed since width:%d height:%d was not set correctly", - XCAM_STR(get_name()), _output_width, _output_height); // aligned at least XCAM_BLENDER_ALIGNED_WIDTH uint32_t aligned_width = XCAM_MAX (16, XCAM_BLENDER_ALIGNED_WIDTH); @@ -397,35 +450,43 @@ CLImage360Stitch::ensure_fisheye_parameters ( SmartPtr<DrmBoBuffer> &input, SmartPtr<DrmBoBuffer> &output) { static bool is_fisheye_inited = false; - XCamReturn ret = XCAM_RETURN_NO_ERROR; if (!is_fisheye_inited) { calc_fisheye_initial_info (output); is_fisheye_inited = true; } - if (!_fisheye[0].pool.ptr ()) - create_buffer_pool (_fisheye[0].pool, _fisheye[0].width, _fisheye[0].height); - if (!_fisheye[1].pool.ptr ()) - create_buffer_pool (_fisheye[1].pool, _fisheye[1].width, _fisheye[1].height); - - _fisheye[0].buf = _fisheye[0].pool->get_buffer (_fisheye[0].pool).dynamic_cast_ptr<DrmBoBuffer> (); - _fisheye[1].buf = _fisheye[1].pool->get_buffer (_fisheye[1].pool).dynamic_cast_ptr<DrmBoBuffer> (); - XCAM_ASSERT (_fisheye[0].buf.ptr () && _fisheye[1].buf.ptr ()); - - ret = ensure_handler_parameters (_fisheye[0].handler, input, _fisheye[0].buf); - STITCH_CHECK (ret, "execute first fisheye prepare_parameters failed"); - ret = ensure_handler_parameters (_fisheye[1].handler, input, _fisheye[1].buf); - STITCH_CHECK (ret, "execute second fisheye prepare_parameters failed"); + SmartPtr<DrmBoBuffer> pre_buf; + SmartPtr<DrmBoBuffer> cur_buf = input; + for (int i = 0; i < _fisheye_num; i++) { + if (!_fisheye[i].pool.ptr ()) + create_buffer_pool (_fisheye[i].pool, _fisheye[i].width, _fisheye[i].height); + + _fisheye[i].buf = _fisheye[i].pool->get_buffer (_fisheye[i].pool).dynamic_cast_ptr<DrmBoBuffer> (); + XCAM_ASSERT (_fisheye[i].buf.ptr ()); + + XCamReturn ret = ensure_handler_parameters (_fisheye[i].handler, cur_buf, _fisheye[i].buf); + STITCH_CHECK (ret, "execute fisheye prepare_parameters failed"); + + if (!_all_in_one_img) { + pre_buf = cur_buf; + cur_buf = cur_buf->find_typed_attach<DrmBoBuffer> (); + if (!cur_buf.ptr () && (i != (_fisheye_num - 1))) { + XCAM_LOG_ERROR ("conflicting attached buffers and fisheye number"); + return XCAM_RETURN_ERROR_PARAM; + } + pre_buf->detach_buffer (cur_buf); + } + } return XCAM_RETURN_NO_ERROR; } XCamReturn CLImage360Stitch::prepare_global_scale_blender_parameters ( - SmartPtr<DrmBoBuffer> &input0, SmartPtr<DrmBoBuffer> &input1, SmartPtr<DrmBoBuffer> &output) + SmartPtr<DrmBoBuffer> &input0, SmartPtr<DrmBoBuffer> &input1, SmartPtr<DrmBoBuffer> &output, + int idx, int idx_next, int &cur_start_pos) { - XCamReturn ret = XCAM_RETURN_NO_ERROR; const VideoBufferInfo &in0_info = input0->get_video_info (); const VideoBufferInfo &in1_info = input1->get_video_info (); const VideoBufferInfo &out_info = output->get_video_info (); @@ -433,76 +494,51 @@ CLImage360Stitch::prepare_global_scale_blender_parameters ( XCAM_ASSERT (in0_info.height == in1_info.height); XCAM_ASSERT (in0_info.width <= out_info.width && in1_info.width <= out_info.width); - Rect main_left = get_image_overlap (ImageIdxMain, 0); - Rect main_right = get_image_overlap (ImageIdxMain, 1); - Rect scnd_left = get_image_overlap (ImageIdxSecondary, 1); - Rect scnd_right = get_image_overlap (ImageIdxSecondary, 0); - int main_mid = XCAM_ALIGN_DOWN (in0_info.width / 2, XCAM_BLENDER_ALIGNED_WIDTH); - int scnd_mid = 0; - int out_mid = XCAM_ALIGN_DOWN (out_info.width / 2, XCAM_BLENDER_ALIGNED_WIDTH); - Rect area, out_merge_window; - area.pos_y = out_merge_window.pos_y = 0; - area.height = out_merge_window.pos_y = out_info.height; - - //calculate left stitching area(input) - int32_t prev_pos = main_left.pos_x; - main_left.pos_x = XCAM_ALIGN_AROUND (main_left.pos_x, XCAM_BLENDER_ALIGNED_WIDTH); - main_left.width = XCAM_ALIGN_UP (main_left.width, XCAM_BLENDER_ALIGNED_WIDTH); - scnd_left.pos_x += main_left.pos_x - prev_pos; - scnd_left.pos_x = XCAM_ALIGN_AROUND (scnd_left.pos_x, XCAM_BLENDER_ALIGNED_WIDTH); - scnd_left.width = main_left.width; - - //calculate right stitching area(input) - prev_pos = main_right.pos_x; - main_right.pos_x = XCAM_ALIGN_AROUND (main_right.pos_x, XCAM_BLENDER_ALIGNED_WIDTH); - main_right.width = XCAM_ALIGN_UP (main_right.width, XCAM_BLENDER_ALIGNED_WIDTH); - scnd_right.pos_x += main_right.pos_x - prev_pos; - scnd_right.pos_x = XCAM_ALIGN_AROUND (scnd_right.pos_x, XCAM_BLENDER_ALIGNED_WIDTH); - scnd_right.width = main_right.width; - - //find scnd_mid - scnd_mid = scnd_left.pos_x + (main_mid - main_left.pos_x) - out_mid; - if (scnd_mid < scnd_right.pos_x + scnd_right.width) - scnd_mid = scnd_right.pos_x + scnd_right.width; - - // set left blender - area.pos_x = scnd_mid; - area.width = scnd_left.pos_x + scnd_left.width - scnd_mid; - _left_blender->set_input_valid_area (area, 0); - - area.pos_x = main_left.pos_x; - area.width = main_mid - main_left.pos_x; - _left_blender->set_input_valid_area (area, 1); - - out_merge_window.width = main_left.width; - out_merge_window.pos_x = out_mid - (main_mid - main_left.pos_x); - _left_blender->set_merge_window (out_merge_window); - _left_blender->set_input_merge_area (scnd_left, 0); - _left_blender->set_input_merge_area (main_left, 1); - - // set right blender - area.pos_x = main_mid; - area.width = main_right.pos_x + main_right.width - main_mid; - _right_blender->set_input_valid_area (area, 0); - - area.pos_x = scnd_right.pos_x; - area.width = scnd_mid - scnd_right.pos_x; - _right_blender->set_input_valid_area (area, 1); - - out_merge_window.pos_x = out_mid + (main_right.pos_x - main_mid); - out_merge_window.width = main_right.width; - _right_blender->set_merge_window (out_merge_window); - _right_blender->set_input_merge_area (main_right, 0); - _right_blender->set_input_merge_area (scnd_right, 1); - - return ret; + Rect left_lap = get_image_overlap (idx, 1); + Rect right_lap = get_image_overlap (idx_next, 0); + + int left_img_mid = XCAM_ALIGN_DOWN (in0_info.width / 2, XCAM_BLENDER_ALIGNED_WIDTH); + int right_img_mid = XCAM_ALIGN_DOWN (in1_info.width / 2, XCAM_BLENDER_ALIGNED_WIDTH); + + int32_t prev_pos; + prev_pos = left_lap.pos_x; + left_lap.pos_x = XCAM_ALIGN_AROUND (left_lap.pos_x, XCAM_BLENDER_ALIGNED_WIDTH); + left_lap.width = XCAM_ALIGN_UP (left_lap.width, XCAM_BLENDER_ALIGNED_WIDTH); + right_lap.pos_x += left_lap.pos_x - prev_pos; + right_lap.pos_x = XCAM_ALIGN_AROUND (right_lap.pos_x, XCAM_BLENDER_ALIGNED_WIDTH); + right_lap.width = left_lap.width; + + Rect area; + area.pos_y = left_lap.pos_y; + area.height = left_lap.height; + area.pos_x = left_img_mid; + area.width = left_lap.pos_x + left_lap.width - left_img_mid; + _blender[idx]->set_input_valid_area (area, 0); + + area.pos_y = right_lap.pos_y; + area.height = right_lap.height; + area.pos_x = right_lap.pos_x; + area.width = right_img_mid - right_lap.pos_x; + _blender[idx]->set_input_valid_area (area, 1); + + Rect out_merge_window; + out_merge_window.width = left_lap.width; + out_merge_window.pos_x = cur_start_pos + (left_lap.pos_x - left_img_mid); + out_merge_window.pos_y = 0; + out_merge_window.height = out_info.height; + _blender[idx]->set_merge_window (out_merge_window); + + _blender[idx]->set_input_merge_area (left_lap, 0); + _blender[idx]->set_input_merge_area (right_lap, 1); + + cur_start_pos += left_lap.pos_x - left_img_mid + right_img_mid - right_lap.pos_x; + return XCAM_RETURN_NO_ERROR; } XCamReturn CLImage360Stitch::prepare_local_scale_blender_parameters ( - SmartPtr<DrmBoBuffer> &input0, SmartPtr<DrmBoBuffer> &input1, SmartPtr<DrmBoBuffer> &output) + SmartPtr<DrmBoBuffer> &input0, SmartPtr<DrmBoBuffer> &input1, SmartPtr<DrmBoBuffer> &output, int idx, int idx_next) { - XCamReturn ret = XCAM_RETURN_NO_ERROR; const VideoBufferInfo &in0_info = input0->get_video_info (); const VideoBufferInfo &in1_info = input1->get_video_info (); const VideoBufferInfo &out_info = output->get_video_info (); @@ -510,67 +546,47 @@ CLImage360Stitch::prepare_local_scale_blender_parameters ( XCAM_ASSERT (in0_info.height == in1_info.height); XCAM_ASSERT (in0_info.width <= out_info.width && in1_info.width <= out_info.width); - Rect main_left = get_image_overlap (ImageIdxMain, 0); - Rect main_right = get_image_overlap (ImageIdxMain, 1); - Rect scnd_left = get_image_overlap (ImageIdxSecondary, 1); - Rect scnd_right = get_image_overlap (ImageIdxSecondary, 0); - - int main_mid = XCAM_ALIGN_DOWN (in0_info.width / 2, XCAM_BLENDER_ALIGNED_WIDTH); - int scnd_mid = XCAM_ALIGN_DOWN (in1_info.width / 2, XCAM_BLENDER_ALIGNED_WIDTH); - int out_mid = XCAM_ALIGN_DOWN (out_info.width / 2, XCAM_BLENDER_ALIGNED_WIDTH); - Rect area, out_merge_window; - area.pos_y = out_merge_window.pos_y = 0; - area.height = out_merge_window.pos_y = out_info.height; - - //calculate left stitching area(input) - int32_t prev_pos = main_left.pos_x; - main_left.pos_x = XCAM_ALIGN_AROUND (main_left.pos_x, XCAM_BLENDER_ALIGNED_WIDTH); - main_left.width = XCAM_ALIGN_UP (main_left.width, XCAM_BLENDER_ALIGNED_WIDTH); - scnd_left.pos_x += main_left.pos_x - prev_pos; - scnd_left.pos_x = XCAM_ALIGN_AROUND (scnd_left.pos_x, XCAM_BLENDER_ALIGNED_WIDTH); - scnd_left.width = main_left.width; - - //calculate right stitching area(input) - prev_pos = main_right.pos_x; - main_right.pos_x = XCAM_ALIGN_AROUND (main_right.pos_x, XCAM_BLENDER_ALIGNED_WIDTH); - main_right.width = XCAM_ALIGN_UP (main_right.width, XCAM_BLENDER_ALIGNED_WIDTH); - scnd_right.pos_x += main_right.pos_x - prev_pos; - scnd_right.pos_x = XCAM_ALIGN_AROUND (scnd_right.pos_x, XCAM_BLENDER_ALIGNED_WIDTH); - scnd_right.width = main_right.width; - - // set left blender - area.pos_x = scnd_mid; - area.width = scnd_left.pos_x + scnd_left.width - scnd_mid; - _left_blender->set_input_valid_area (area, 0); - - area.pos_x = main_left.pos_x; - area.width = main_mid - main_left.pos_x; - _left_blender->set_input_valid_area (area, 1); - - int delta_width = out_mid - (main_mid - main_left.pos_x) - (scnd_left.pos_x - scnd_mid); - out_merge_window.width = main_left.width + delta_width; - out_merge_window.pos_x = scnd_left.pos_x - scnd_mid; - _left_blender->set_merge_window (out_merge_window); - _left_blender->set_input_merge_area (scnd_left, 0); - _left_blender->set_input_merge_area (main_left, 1); - - // set right blender - area.pos_x = main_mid; - area.width = main_right.pos_x + main_right.width - main_mid; - _right_blender->set_input_valid_area (area, 0); - - area.pos_x = scnd_right.pos_x; - area.width = scnd_mid - scnd_right.pos_x; - _right_blender->set_input_valid_area (area, 1); - - delta_width = out_mid - (scnd_mid - scnd_right.pos_x) - (main_right.pos_x - main_mid); - out_merge_window.width = main_right.width + delta_width; - out_merge_window.pos_x = out_mid + (main_right.pos_x - main_mid); - _right_blender->set_merge_window (out_merge_window); - _right_blender->set_input_merge_area (main_right, 0); - _right_blender->set_input_merge_area (scnd_right, 1); - - return ret; + Rect left_lap = get_image_overlap (idx, 1); + Rect right_lap = get_image_overlap (idx_next, 0); + + int left_img_mid = XCAM_ALIGN_DOWN (in0_info.width / 2, XCAM_BLENDER_ALIGNED_WIDTH); + int right_img_mid = XCAM_ALIGN_DOWN (in1_info.width / 2, XCAM_BLENDER_ALIGNED_WIDTH); + int cur_start_pos = XCAM_ALIGN_DOWN (out_info.width / _fisheye_num * idx, XCAM_BLENDER_ALIGNED_WIDTH); + int merge_std_width = XCAM_ALIGN_DOWN (out_info.width / _fisheye_num, XCAM_BLENDER_ALIGNED_WIDTH); + + int32_t prev_pos; + prev_pos = left_lap.pos_x; + left_lap.pos_x = XCAM_ALIGN_AROUND (left_lap.pos_x, XCAM_BLENDER_ALIGNED_WIDTH); + left_lap.width = XCAM_ALIGN_UP (left_lap.width, XCAM_BLENDER_ALIGNED_WIDTH); + right_lap.pos_x += left_lap.pos_x - prev_pos; + right_lap.pos_x = XCAM_ALIGN_AROUND (right_lap.pos_x, XCAM_BLENDER_ALIGNED_WIDTH); + right_lap.width = left_lap.width; + + Rect area; + area.pos_y = left_lap.pos_y; + area.height = left_lap.height; + area.pos_x = left_img_mid; + area.width = left_lap.pos_x + left_lap.width - left_img_mid; + _blender[idx]->set_input_valid_area (area, 0); + + area.pos_y = right_lap.pos_y; + area.height = right_lap.height; + area.pos_x = right_lap.pos_x; + area.width = right_img_mid - right_lap.pos_x; + _blender[idx]->set_input_valid_area (area, 1); + + Rect out_merge_window; + int delta_width = merge_std_width - (right_img_mid - right_lap.pos_x) - (left_lap.pos_x - left_img_mid); + out_merge_window.width = left_lap.width + delta_width; + out_merge_window.pos_x = cur_start_pos + (left_lap.pos_x - left_img_mid); + out_merge_window.pos_y = 0; + out_merge_window.height = out_info.height; + _blender[idx]->set_merge_window (out_merge_window); + + _blender[idx]->set_input_merge_area (left_lap, 0); + _blender[idx]->set_input_merge_area (right_lap, 1); + + return XCAM_RETURN_NO_ERROR; } bool @@ -599,12 +615,14 @@ CLImage360Stitch::reset_buffer_info (SmartPtr<DrmBoBuffer> &input) VideoBufferInfo reset_info; const VideoBufferInfo &buf_info = input->get_video_info (); - Rect img0_left = get_image_overlap (ImageIdxMain, 0); - Rect img0_right = get_image_overlap (ImageIdxMain, 1); - Rect img1_left = get_image_overlap (ImageIdxSecondary, 0); - Rect img1_right = get_image_overlap (ImageIdxSecondary, 1); + uint32_t reset_width = 0; + for (int i = 0; i < _fisheye_num; i++) { + Rect img_left = get_image_overlap (i, 0); + Rect img_right = get_image_overlap (i, 1); + + reset_width += img_right.pos_x - img_left.pos_x; + } - uint32_t reset_width = img0_right.pos_x - img0_left.pos_x + img1_right.pos_x - img1_left.pos_x; reset_width = XCAM_ALIGN_UP (reset_width, XCAM_BLENDER_ALIGNED_WIDTH); reset_info.init (buf_info.format, reset_width, buf_info.height, buf_info.aligned_width, buf_info.aligned_height); @@ -623,17 +641,21 @@ CLImage360Stitch::prepare_parameters (SmartPtr<DrmBoBuffer> &input, SmartPtr<Drm ret = ensure_fisheye_parameters (input, output); STITCH_CHECK (ret, "ensure fisheye parameters failed"); - _fisheye[0].buf->attach_buffer (_fisheye[1].buf); - update_image_overlap (); if (_scale_mode == CLBlenderScaleLocal) { - ret = prepare_local_scale_blender_parameters (_fisheye[0].buf, _fisheye[1].buf, output); - STITCH_CHECK (ret, "prepare local scale blender parameters failed"); - - ret = ensure_handler_parameters (_left_blender, _fisheye[0].buf, output); - STITCH_CHECK (ret, "left blender: execute ensure_parameters failed"); - ret = ensure_handler_parameters (_right_blender, _fisheye[0].buf, output); - STITCH_CHECK (ret, "right blender: execute ensure_parameters failed"); + int idx_next = 1; + for (int i = 0; i < _fisheye_num; i++) { + idx_next = (i == (_fisheye_num - 1)) ? 0 : (i + 1); + + ret = prepare_local_scale_blender_parameters ( + _fisheye[i].buf, _fisheye[idx_next].buf, output, i, idx_next); + STITCH_CHECK (ret, "prepare local scale blender parameters failed"); + + _fisheye[i].buf->attach_buffer (_fisheye[idx_next].buf); + ret = ensure_handler_parameters (_blender[i], _fisheye[i].buf, output); + STITCH_CHECK (ret, "blender: execute ensure_parameters failed"); + _fisheye[i].buf->detach_buffer (_fisheye[idx_next].buf); + } } else { //global scale const VideoBufferInfo &buf_info = output->get_video_info (); if (!_scale_buf_pool.ptr ()) @@ -642,13 +664,20 @@ CLImage360Stitch::prepare_parameters (SmartPtr<DrmBoBuffer> &input, SmartPtr<Drm _scale_buf_pool->get_buffer (_scale_buf_pool).dynamic_cast_ptr<DrmBoBuffer> (); XCAM_ASSERT (scale_input.ptr ()); - ret = prepare_global_scale_blender_parameters (_fisheye[0].buf, _fisheye[1].buf, scale_input); - STITCH_CHECK (ret, "prepare global scale blender parameters failed"); + int idx_next = 1; + int cur_start_pos = 0; + for (int i = 0; i < _fisheye_num; i++) { + idx_next = (i == (_fisheye_num - 1)) ? 0 : (i + 1); + + ret = prepare_global_scale_blender_parameters ( + _fisheye[i].buf, _fisheye[idx_next].buf, scale_input, i, idx_next, cur_start_pos); + STITCH_CHECK (ret, "prepare global scale blender parameters failed"); - ret = ensure_handler_parameters (_left_blender, _fisheye[0].buf, scale_input); - STITCH_CHECK (ret, "left blender: execute ensure_parameters failed"); - ret = ensure_handler_parameters (_right_blender, _fisheye[0].buf, scale_input); - STITCH_CHECK (ret, "right blender: execute ensure_parameters failed"); + _fisheye[i].buf->attach_buffer (_fisheye[idx_next].buf); + ret = ensure_handler_parameters (_blender[i], _fisheye[i].buf, scale_input); + STITCH_CHECK (ret, "blender: execute ensure_parameters failed"); + _fisheye[i].buf->detach_buffer (_fisheye[idx_next].buf); + } reset_buffer_info (scale_input); _scale_global_input = scale_input; @@ -662,8 +691,12 @@ XCamReturn CLImage360Stitch::execute_done (SmartPtr<DrmBoBuffer> &output) { #if HAVE_OPENCV - if (!_feature_match->is_ocl_path ()) - get_context ()->finish (); + for (int i = 0; i < _fisheye_num; i++) { + if (!_feature_match[i]->is_ocl_path ()) { + get_context ()->finish (); + break; + } + } #endif _scale_global_input.release (); @@ -674,26 +707,19 @@ CLImage360Stitch::execute_done (SmartPtr<DrmBoBuffer> &output) #if HAVE_OPENCV static void -convert_to_cv_rect (ImageMergeInfo merge_info, cv::Rect &crop_left, cv::Rect &crop_right) +convert_to_cv_rect (Rect xcam_rect, cv::Rect &cv_rect) { - crop_left.x = merge_info.left.pos_x; - crop_left.y = merge_info.left.pos_y + merge_info.left.height / 3; - crop_left.width = merge_info.left.width; - crop_left.height = merge_info.left.height / 3; - - crop_right.x = merge_info.right.pos_x; - crop_right.y = merge_info.right.pos_y + merge_info.right.height / 3; - crop_right.width = merge_info.right.width; - crop_right.height = merge_info.right.height / 3; + cv_rect.x = xcam_rect.pos_x; + cv_rect.y = xcam_rect.pos_y + xcam_rect.height / 3; + cv_rect.width = xcam_rect.width; + cv_rect.height = xcam_rect.height / 3; } static void -convert_to_xcam_rect (cv::Rect crop_left, cv::Rect crop_right, ImageMergeInfo &merge_info) +convert_to_xcam_rect (cv::Rect cv_rect, Rect &xcam_rect) { - merge_info.left.pos_x = crop_left.x; - merge_info.left.width = crop_left.width; - merge_info.right.pos_x = crop_right.x; - merge_info.right.width = crop_right.width; + xcam_rect.pos_x = cv_rect.x; + xcam_rect.width = cv_rect.width; } #endif @@ -703,20 +729,22 @@ CLImage360Stitch::sub_handler_execute_done (SmartPtr<CLImageHandler> &handler) #if HAVE_OPENCV XCAM_ASSERT (handler.ptr ()); - if (handler.ptr () == _fisheye[ImageIdxCount - 1].handler.ptr ()) { - cv::Rect img0_crop_left, img0_crop_right, img1_crop_left, img1_crop_right; + if (handler.ptr () == _fisheye[_fisheye_num - 1].handler.ptr ()) { + int idx_next = 1; + cv::Rect crop_left, crop_right; + + for (int i = 0; i < _fisheye_num; i++) { + idx_next = (i == (_fisheye_num - 1)) ? 0 : (i + 1); - convert_to_cv_rect (_img_merge_info[0], img0_crop_left, img0_crop_right); - convert_to_cv_rect (_img_merge_info[1], img1_crop_left, img1_crop_right); - get_context ()->finish (); + convert_to_cv_rect (_img_merge_info[i].right, crop_left); + convert_to_cv_rect (_img_merge_info[idx_next].left, crop_right); - _feature_match->optical_flow_feature_match ( - _fisheye[0].width, _fisheye[0].buf, _fisheye[1].buf, - img0_crop_left, img0_crop_right, img1_crop_left, img1_crop_right); + _feature_match[i]->optical_flow_feature_match ( + _fisheye[i].buf, _fisheye[idx_next].buf, crop_left, crop_right, _fisheye[i].width); - //update merge info - convert_to_xcam_rect (img0_crop_left, img0_crop_right, _img_merge_info[0]); - convert_to_xcam_rect (img1_crop_left, img1_crop_right, _img_merge_info[1]); + convert_to_xcam_rect (crop_left, _img_merge_info[i].right); + convert_to_xcam_rect (crop_right, _img_merge_info[idx_next].left); + } } #else XCAM_UNUSED (handler); @@ -755,32 +783,30 @@ create_blender_global_scale_kernel ( SmartPtr<CLImageHandler> create_image_360_stitch ( const SmartPtr<CLContext> &context, bool need_seam, - CLBlenderScaleMode scale_mode, bool fisheye_map, bool need_lsc, StitchResMode res_mode) + CLBlenderScaleMode scale_mode, bool fisheye_map, bool need_lsc, StitchResMode res_mode, + int fisheye_num, bool all_in_one_img) { const int layer = 2; const bool need_uv = true; SmartPtr<CLFisheyeHandler> fisheye; - SmartPtr<CLBlender> left_blender, right_blender; - SmartPtr<CLImage360Stitch> stitch = new CLImage360Stitch (context, scale_mode, res_mode); + SmartPtr<CLBlender> blender; + SmartPtr<CLImage360Stitch> stitch = new CLImage360Stitch ( + context, scale_mode, res_mode, fisheye_num, all_in_one_img); XCAM_ASSERT (stitch.ptr ()); - for (int index = 0; index < ImageIdxCount; ++index) { + for (int index = 0; index < fisheye_num; ++index) { fisheye = create_fisheye_handler (context, fisheye_map, need_lsc).dynamic_cast_ptr<CLFisheyeHandler> (); XCAM_FAIL_RETURN (ERROR, fisheye.ptr (), NULL, "image_360_stitch create fisheye handler failed"); fisheye->disable_buf_pool (true); stitch->set_fisheye_handler (fisheye, index); } - left_blender = create_pyramid_blender (context, layer, need_uv, need_seam, scale_mode).dynamic_cast_ptr<CLBlender> (); - XCAM_FAIL_RETURN (ERROR, left_blender.ptr (), NULL, "image_360_stitch create left blender failed"); - left_blender->disable_buf_pool (true); - left_blender->swap_input_idx (true); - stitch->set_left_blender (left_blender); - - right_blender = create_pyramid_blender (context, layer, need_uv, need_seam, scale_mode).dynamic_cast_ptr<CLBlender> (); - XCAM_FAIL_RETURN (ERROR, right_blender.ptr (), NULL, "image_360_stitch create right blender failed"); - right_blender->disable_buf_pool (true); - stitch->set_right_blender (right_blender); + for (int index = 0; index < fisheye_num; ++index) { + blender = create_pyramid_blender (context, layer, need_uv, need_seam, scale_mode).dynamic_cast_ptr<CLBlender> (); + XCAM_FAIL_RETURN (ERROR, blender.ptr (), NULL, "image_360_stitch create blender failed"); + blender->disable_buf_pool (true); + stitch->set_blender (blender, index); + } if (scale_mode == CLBlenderScaleGlobal) { int max_plane = need_uv ? 2 : 1; diff --git a/modules/ocl/cl_image_360_stitch.h b/modules/ocl/cl_image_360_stitch.h index 6eee4e7..4ea238c 100644 --- a/modules/ocl/cl_image_360_stitch.h +++ b/modules/ocl/cl_image_360_stitch.h @@ -65,7 +65,8 @@ class CLImage360Stitch { public: explicit CLImage360Stitch ( - const SmartPtr<CLContext> &context, CLBlenderScaleMode scale_mode, StitchResMode res_mode); + const SmartPtr<CLContext> &context, CLBlenderScaleMode scale_mode, StitchResMode res_mode, + int fisheye_num, bool all_in_one_img); bool set_stitch_info (StitchInfo stitch_info); StitchInfo get_stitch_info (); @@ -75,13 +76,12 @@ public: } bool set_fisheye_handler (SmartPtr<CLFisheyeHandler> fisheye, int index); - bool set_left_blender (SmartPtr<CLBlender> blender); - bool set_right_blender (SmartPtr<CLBlender> blender); + bool set_blender (SmartPtr<CLBlender> blender, int idx); bool set_image_overlap (const int idx, const Rect &overlap0, const Rect &overlap1); - const Rect &get_image_overlap (ImageIdx image, int num) { - XCAM_ASSERT (image < ImageIdxCount && num < 2); - return _overlaps[image][num]; + const Rect &get_image_overlap (int img_idx, int num) { + XCAM_ASSERT (img_idx < _fisheye_num && num < 2); + return _overlaps[img_idx][num]; } SmartPtr<DrmBoBuffer> &get_global_scale_input () { @@ -93,8 +93,8 @@ public: void set_feature_match_ocl (bool use_ocl); #if HAVE_OPENCV - void set_feature_match_config (CVFMConfig config); - CVFMConfig get_feature_match_config (); + void set_feature_match_config (const int idx, CVFMConfig config); + CVFMConfig get_feature_match_config (const int idx); #endif protected: @@ -104,9 +104,10 @@ protected: XCamReturn ensure_fisheye_parameters (SmartPtr<DrmBoBuffer> &input, SmartPtr<DrmBoBuffer> &output); XCamReturn prepare_local_scale_blender_parameters ( - SmartPtr<DrmBoBuffer> &input0, SmartPtr<DrmBoBuffer> &input1, SmartPtr<DrmBoBuffer> &output); + SmartPtr<DrmBoBuffer> &input0, SmartPtr<DrmBoBuffer> &input1, SmartPtr<DrmBoBuffer> &output, int idx, int idx_next); XCamReturn prepare_global_scale_blender_parameters ( - SmartPtr<DrmBoBuffer> &input0, SmartPtr<DrmBoBuffer> &input1, SmartPtr<DrmBoBuffer> &output); + SmartPtr<DrmBoBuffer> &input0, SmartPtr<DrmBoBuffer> &input1, SmartPtr<DrmBoBuffer> &output, + int idx, int idx_next, int &cur_start_pos); bool create_buffer_pool (SmartPtr<BufferPool> &buf_pool, uint32_t width, uint32_t height); XCamReturn reset_buffer_info (SmartPtr<DrmBoBuffer> &input); @@ -121,17 +122,16 @@ private: private: SmartPtr<CLContext> _context; - CLFisheyeParams _fisheye[ImageIdxCount]; - SmartPtr<CLBlender> _left_blender; - SmartPtr<CLBlender> _right_blender; + CLFisheyeParams _fisheye[XCAM_STITCH_FISHEYE_MAX_NUM]; + SmartPtr<CLBlender> _blender[XCAM_STITCH_FISHEYE_MAX_NUM]; #if HAVE_OPENCV - SmartPtr<CVFeatureMatch> _feature_match; + SmartPtr<CVFeatureMatch> _feature_match[XCAM_STITCH_FISHEYE_MAX_NUM]; #endif uint32_t _output_width; uint32_t _output_height; - ImageMergeInfo _img_merge_info[ImageIdxCount]; - Rect _overlaps[ImageIdxCount][2]; // 2=>Overlap0 and overlap1 + ImageMergeInfo _img_merge_info[XCAM_STITCH_FISHEYE_MAX_NUM]; + Rect _overlaps[XCAM_STITCH_FISHEYE_MAX_NUM][2]; // 2=>Overlap0 and overlap1 CLBlenderScaleMode _scale_mode; SmartPtr<BufferPool> _scale_buf_pool; @@ -141,6 +141,8 @@ private: StitchResMode _res_mode; bool _is_stitch_inited; + int _fisheye_num; + bool _all_in_one_img; StitchInfo _stitch_info; }; @@ -151,7 +153,9 @@ create_image_360_stitch ( CLBlenderScaleMode scale_mode = CLBlenderScaleLocal, bool fisheye_map = false, bool need_lsc = false, - StitchResMode res_mode = StitchRes1080P); + StitchResMode res_mode = StitchRes1080P, + int fisheye_num = 2, + bool all_in_one_img = true); } diff --git a/modules/ocl/cl_pyramid_blender.cpp b/modules/ocl/cl_pyramid_blender.cpp index c83e8f1..fc90179 100644 --- a/modules/ocl/cl_pyramid_blender.cpp +++ b/modules/ocl/cl_pyramid_blender.cpp @@ -322,8 +322,6 @@ PyramidLayer::bind_buf_to_layer0 ( uint32_t divider_vert[2] = {1, 2}; XCAM_ASSERT (in0_info.height == in1_info.height); - XCAM_ASSERT (in0_info.width + in1_info.width >= out_info.width); - //XCAM_ASSERT (out_info.height == in0_info.height); XCAM_ASSERT (merge0_rect.width == merge1_rect.width); this->blend_width = XCAM_ALIGN_UP (merge0_rect.width, XCAM_BLENDER_ALIGNED_WIDTH); diff --git a/modules/ocl/cv_feature_match.cpp b/modules/ocl/cv_feature_match.cpp index 0a526d5..a3ae34c 100644 --- a/modules/ocl/cv_feature_match.cpp +++ b/modules/ocl/cv_feature_match.cpp @@ -46,7 +46,6 @@ dump_buffer (SmartPtr<DrmBoBuffer> buffer, char *dump_name) } file.close (); - XCAM_LOG_INFO ("write buffer to %s done", dump_name); return XCAM_RETURN_NO_ERROR; } @@ -54,10 +53,12 @@ dump_buffer (SmartPtr<DrmBoBuffer> buffer, char *dump_name) CVFeatureMatch::CVFeatureMatch () : CVBaseClass() + , _x_offset (0.0f) + , _mean_offset (0.0f) + , _valid_count (0) + , _fm_idx (-1) + , _frame_num (0) { - xcam_mem_clear (_x_offset); - xcam_mem_clear (_mean_offset); - xcam_mem_clear (_valid_count); } void @@ -72,11 +73,15 @@ CVFeatureMatch::get_config () return _config; } +void +CVFeatureMatch::set_fm_index (int idx) +{ + _fm_idx = idx; +} + bool CVFeatureMatch::get_crop_image ( - SmartPtr<DrmBoBuffer> buffer, - cv::Rect img_crop_left, cv::Rect img_crop_right, - cv::UMat &img_left, cv::UMat &img_right) + SmartPtr<DrmBoBuffer> buffer, cv::Rect img_crop, cv::UMat &img) { SmartPtr<CLBuffer> cl_buffer = new CLVaBuffer (_context, buffer); VideoBufferInfo info = buffer->get_video_info (); @@ -89,14 +94,14 @@ CVFeatureMatch::get_crop_image ( return false; } - img_left = umat (img_crop_left); - img_right = umat (img_crop_right); + img = umat (img_crop); return true; } void -CVFeatureMatch::add_detected_data (cv::InputArray image, cv::Ptr<cv::Feature2D> detector, std::vector<cv::Point2f> &corners) +CVFeatureMatch::add_detected_data ( + cv::InputArray image, cv::Ptr<cv::Feature2D> detector, std::vector<cv::Point2f> &corners) { std::vector<cv::KeyPoint> keypoints; detector->detect (image, keypoints); @@ -203,7 +208,7 @@ CVFeatureMatch::calc_of_match ( cv::InputArray image0, cv::InputArray image1, std::vector<cv::Point2f> corner0, std::vector<cv::Point2f> corner1, std::vector<uchar> &status, std::vector<float> &error, - int &last_count, float &last_mean_offset, float &out_x_offset, int frame_num, int idx) + int &last_count, float &last_mean_offset, float &out_x_offset) { cv::_InputOutputArray out_image; cv::Size img0_size = image0.size (); @@ -258,12 +263,8 @@ CVFeatureMatch::calc_of_match ( #if XCAM_CV_FM_DEBUG char file_name[1024]; - snprintf (file_name, 1023, "fm_optical_flow_%d_%d.jpg", frame_num, idx); + snprintf (file_name, 1023, "fm_optical_flow_%d_%d.jpg", _frame_num, _fm_idx); cv::imwrite (file_name, out_image); - XCAM_LOG_INFO ("write feature match: %s", file_name); -#else - XCAM_UNUSED (frame_num); - XCAM_UNUSED (idx); #endif } @@ -300,8 +301,6 @@ CVFeatureMatch::detect_and_match ( std::vector<cv::Point2f> corner_left, corner_right; cv::Ptr<cv::Feature2D> fast_detector; cv::Size win_size = cv::Size (5, 5); - static int idx = 0; - static int frame_num = 0; if (img_left.isUMat ()) win_size = cv::Size (16, 16); @@ -310,11 +309,6 @@ CVFeatureMatch::detect_and_match ( add_detected_data (img_left, fast_detector, corner_left); if (corner_left.empty ()) { -#if XCAM_CV_FM_DEBUG - if (idx == 1) - frame_num++; - idx = (idx == 0) ? 1 : 0; -#endif return; } @@ -324,87 +318,72 @@ CVFeatureMatch::detect_and_match ( cv::ocl::finish(); calc_of_match (img_left, img_right, corner_left, corner_right, - status, err, valid_count, mean_offset, x_offset, frame_num, idx); + status, err, valid_count, mean_offset, x_offset); adjust_stitch_area (dst_width, x_offset, crop_left, crop_right); #if XCAM_CV_FM_DEBUG XCAM_LOG_INFO ( - "Stiching area %d: left_area(x:%d, width:%d), right_area(x:%d, width:%d)", - idx, crop_left.x, crop_left.width, crop_right.x, crop_right.width); - - if (idx == 1) - frame_num++; - idx = (idx == 0) ? 1 : 0; + "Stiching area: left_area(x:%d, width:%d), right_area(x:%d, width:%d)", + crop_left.x, crop_left.width, crop_right.x, crop_right.width); #endif } void CVFeatureMatch::optical_flow_feature_match ( - int dst_width, SmartPtr<DrmBoBuffer> buf0, SmartPtr<DrmBoBuffer> buf1, - cv::Rect &img0_crop_left, cv::Rect &img0_crop_right, cv::Rect &img1_crop_left, cv::Rect &img1_crop_right) + SmartPtr<DrmBoBuffer> left_buf, SmartPtr<DrmBoBuffer> right_buf, + cv::Rect &left_img_crop, cv::Rect &right_img_crop, int dst_width) { - cv::UMat umat0_left, umat0_right, umat1_left, umat1_right; - cv::Mat mat0_left, mat0_right, mat1_left, mat1_right; - cv::_InputArray img0_left, img0_right, img1_left, img1_right; + cv::UMat left_umat, right_umat; + cv::Mat left_mat, right_mat; + cv::_InputArray left_img, right_img; - if (!get_crop_image (buf0, img0_crop_left, img0_crop_right, umat0_left, umat0_right) - || !get_crop_image (buf1, img1_crop_left, img1_crop_right, umat1_left, umat1_right)) + if (!get_crop_image (left_buf, left_img_crop, left_umat) + || !get_crop_image (right_buf, right_img_crop, right_umat)) return; if (_use_ocl) { - img0_left = cv::_InputArray (umat0_left); - img0_right = cv::_InputArray (umat0_right); - img1_left = cv::_InputArray (umat1_left); - img1_right = cv::_InputArray (umat1_right); + left_img = cv::_InputArray (left_umat); + right_img = cv::_InputArray (right_umat); } else { - mat0_left = umat0_left.getMat (cv::ACCESS_READ); - mat0_right = umat0_right.getMat (cv::ACCESS_READ); - mat1_left = umat1_left.getMat (cv::ACCESS_READ); - mat1_right = umat1_right.getMat (cv::ACCESS_READ); - - img0_left = cv::_InputArray (mat0_left); - img0_right = cv::_InputArray (mat0_right); - img1_left = cv::_InputArray (mat1_left); - img1_right = cv::_InputArray (mat1_right); + left_mat = left_umat.getMat (cv::ACCESS_READ); + right_mat = right_umat.getMat (cv::ACCESS_READ); + + left_img = cv::_InputArray (left_mat); + right_img = cv::_InputArray (right_mat); } - detect_and_match (img1_right, img0_left, img1_crop_right, img0_crop_left, - _valid_count[0], _mean_offset[0], _x_offset[0], dst_width); - detect_and_match (img0_right, img1_left, img0_crop_right, img1_crop_left, - _valid_count[1], _mean_offset[1], _x_offset[1], dst_width); + detect_and_match (left_img, right_img, left_img_crop, right_img_crop, + _valid_count, _mean_offset, _x_offset, dst_width); #if XCAM_CV_FM_DEBUG - static int frame = 0; + XCAM_ASSERT (_fm_idx >= 0); char file_name[1024]; - VideoBufferInfo info = buf0->get_video_info (); - - std::snprintf (file_name, 1023, "fm_in_%dx%d_%d_0.nv12", info.width, info.height, frame); - dump_buffer (buf0, file_name); - std::snprintf (file_name, 1023, "fm_in_%dx%d_%d_1.nv12", info.width, info.height, frame); - dump_buffer (buf1, file_name); - - cv::Mat in_mat; - std::snprintf (file_name, 1023, "fm_in_stitch_area_%d_0.jpg", frame); - convert_to_mat (buf0, in_mat); - cv::line (in_mat, cv::Point(img0_crop_left.x, 0), cv::Point(img0_crop_left.x, dst_width), cv::Scalar(0, 0, 255), 2); - cv::line (in_mat, cv::Point(img0_crop_left.x + img0_crop_left.width, 0), - cv::Point(img0_crop_left.x + img0_crop_left.width, dst_width), cv::Scalar(0, 0, 255), 2); - cv::line (in_mat, cv::Point(img0_crop_right.x, 0), cv::Point(img0_crop_right.x, dst_width), cv::Scalar(0, 0, 255), 2); - cv::line (in_mat, cv::Point(img0_crop_right.x + img0_crop_right.width, 0), - cv::Point(img0_crop_right.x + img0_crop_right.width, dst_width), cv::Scalar(0, 0, 255), 2); - cv::imwrite (file_name, in_mat); - - std::snprintf (file_name, 1023, "fm_in_stitch_area_%d_1.jpg", frame); - convert_to_mat (buf1, in_mat); - cv::line (in_mat, cv::Point(img1_crop_left.x, 0), cv::Point(img1_crop_left.x, dst_width), cv::Scalar(0, 0, 255), 2); - cv::line (in_mat, cv::Point(img1_crop_left.x + img1_crop_left.width, 0), - cv::Point(img1_crop_left.x + img1_crop_left.width, dst_width), cv::Scalar(0, 0, 255), 2); - cv::line (in_mat, cv::Point(img1_crop_right.x, 0), cv::Point(img1_crop_right.x, dst_width), cv::Scalar(0, 0, 255), 2); - cv::line (in_mat, cv::Point(img1_crop_right.x + img1_crop_right.width, 0), - cv::Point(img1_crop_right.x + img1_crop_right.width, dst_width), cv::Scalar(0, 0, 255), 2); - cv::imwrite (file_name, in_mat); - - frame++; + + VideoBufferInfo info = left_buf->get_video_info (); + std::snprintf (file_name, 1023, "fm_in_%d_%d_%dx%d_0.nv12", info.width, info.height, _frame_num, _fm_idx); + dump_buffer (left_buf, file_name); + + info = right_buf->get_video_info (); + std::snprintf (file_name, 1023, "fm_in_%d_%d_%dx%d_1.nv12", info.width, info.height, _frame_num, _fm_idx); + dump_buffer (right_buf, file_name); + + cv::Mat mat; + std::snprintf (file_name, 1023, "fm_in_stitch_area_%d_%d_0.jpg", _frame_num, _fm_idx); + convert_to_mat (left_buf, mat); + cv::line (mat, cv::Point(left_img_crop.x, 0), cv::Point(left_img_crop.x, dst_width), cv::Scalar(0, 0, 255), 2); + cv::line (mat, cv::Point(left_img_crop.x + left_img_crop.width, 0), + cv::Point(left_img_crop.x + left_img_crop.width, dst_width), cv::Scalar(0, 0, 255), 2); + cv::imwrite (file_name, mat); + + std::snprintf (file_name, 1023, "fm_in_stitch_area_%d_%d_1.jpg", _frame_num, _fm_idx); + convert_to_mat (right_buf, mat); + cv::line (mat, cv::Point(right_img_crop.x, 0), cv::Point(right_img_crop.x, dst_width), cv::Scalar(0, 0, 255), 2); + cv::line (mat, cv::Point(right_img_crop.x + right_img_crop.width, 0), + cv::Point(right_img_crop.x + right_img_crop.width, dst_width), cv::Scalar(0, 0, 255), 2); + cv::imwrite (file_name, mat); + + XCAM_LOG_INFO ("Feature match: frame number:%d index:%d done", _frame_num, _fm_idx); + _frame_num++; #endif } diff --git a/modules/ocl/cv_feature_match.h b/modules/ocl/cv_feature_match.h index cf65a35..b9117c7 100644 --- a/modules/ocl/cv_feature_match.h +++ b/modules/ocl/cv_feature_match.h @@ -65,24 +65,26 @@ public: void set_config (CVFMConfig config); CVFMConfig get_config (); + void set_fm_index (int idx); + void optical_flow_feature_match ( - int output_width, SmartPtr<DrmBoBuffer> buf0, SmartPtr<DrmBoBuffer> buf1, - cv::Rect &img0_crop_left, cv::Rect &img0_crop_right, cv::Rect &img1_crop_left, cv::Rect &img1_crop_right); + SmartPtr<DrmBoBuffer> left_buf, SmartPtr<DrmBoBuffer> right_buf, + cv::Rect &left_img_crop, cv::Rect &right_img_crop, int dst_width); protected: - bool get_crop_image (SmartPtr<DrmBoBuffer> buffer, - cv::Rect img_crop_left, cv::Rect img_crop_right, cv::UMat &img_left, cv::UMat &img_right); + bool get_crop_image (SmartPtr<DrmBoBuffer> buffer, cv::Rect img_crop, cv::UMat &img); void add_detected_data (cv::InputArray image, cv::Ptr<cv::Feature2D> detector, std::vector<cv::Point2f> &corners); void get_valid_offsets (cv::InputOutputArray out_image, cv::Size img0_size, std::vector<cv::Point2f> corner0, std::vector<cv::Point2f> corner1, - std::vector<uchar> status, std::vector<float> error, std::vector<float> &offsets, float &sum, int &count); + std::vector<uchar> status, std::vector<float> error, + std::vector<float> &offsets, float &sum, int &count); bool get_mean_offset (std::vector<float> offsets, float sum, int &count, float &mean_offset); void calc_of_match (cv::InputArray image0, cv::InputArray image1, std::vector<cv::Point2f> corner0, std::vector<cv::Point2f> corner1, std::vector<uchar> &status, std::vector<float> &error, - int &last_count, float &last_mean_offset, float &out_x_offset, int frame_num, int idx); + int &last_count, float &last_mean_offset, float &out_x_offset); void adjust_stitch_area (int dst_width, float &x_offset, cv::Rect &stitch0, cv::Rect &stitch1); void detect_and_match ( cv::InputArray img_left, cv::InputArray img_right, cv::Rect &crop_left, cv::Rect &crop_right, @@ -94,9 +96,13 @@ private: private: CVFMConfig _config; - float _x_offset[XCAM_CV_FM_MATCH_NUM]; - float _mean_offset[XCAM_CV_FM_MATCH_NUM]; - int _valid_count[XCAM_CV_FM_MATCH_NUM]; + float _x_offset; + float _mean_offset; + int _valid_count; + + // debug parameters + int _fm_idx; + uint _frame_num; }; } diff --git a/tests/test-image-stitching.cpp b/tests/test-image-stitching.cpp index 3475198..f5a3628 100644 --- a/tests/test-image-stitching.cpp +++ b/tests/test-image-stitching.cpp @@ -37,6 +37,13 @@ using namespace XCam; #if HAVE_OPENCV +#if XCAM_TEST_STITCH_DEBUG +static void dbg_write_image ( + SmartPtr<CLContext> context, SmartPtr<CLImage360Stitch> image_360, + SmartPtr<DrmBoBuffer> input_bufs[], SmartPtr<DrmBoBuffer> output_buf, + bool all_in_one, int fisheye_num, int input_count); +#endif + void init_opencv_ocl (SmartPtr<CLContext> context) { @@ -83,7 +90,7 @@ void usage(const char* arg0) "\t--input-h optional, input height, default: 1080\n" "\t--output-w optional, output width, default: 1920\n" "\t--output-h optional, output width, default: 960\n" - "\t--res-mode optional, image resolution mode, select from [1080p/4k], default: 1080p\n" + "\t--res-mode optional, image resolution mode, select from [1080p/1080p4/4k], default: 1080p\n" "\t--scale-mode optional, image scaling mode, select from [local/global], default: local\n" "\t--enable-seam optional, enable seam finder in blending area, default: no\n" "\t--enable-fisheyemap optional, enable fisheye map, default: no\n" @@ -91,6 +98,8 @@ void usage(const char* arg0) #if HAVE_OPENCV "\t--fm-ocl optional, enable ocl for feature match, select from [true/false], default: false\n" #endif + "\t--fisheye-num optional, the number of fisheye lens, default: 2\n" + "\t--all-in-one optional, all fisheye in one image, select from [true/false], default: true\n" "\t--save optional, save file or not, select from [true/false], default: true\n" "\t--framerate optional, framerate of saved video, default: 30.0\n" "\t--loop optional, how many loops need to run for performance test, default: 1\n" @@ -102,10 +111,11 @@ int main (int argc, char *argv[]) { XCamReturn ret = XCAM_RETURN_NO_ERROR; SmartPtr<CLContext> context; - SmartPtr<DrmDisplay> display; - SmartPtr<BufferPool> buf_pool; + SmartPtr<DrmDisplay> display[XCAM_STITCH_FISHEYE_MAX_NUM]; + SmartPtr<BufferPool> buf_pool[XCAM_STITCH_FISHEYE_MAX_NUM]; SmartPtr<VideoBuffer> read_buf; - ImageFileHandle file_in, file_out; + ImageFileHandle file_in[XCAM_STITCH_FISHEYE_MAX_NUM]; + ImageFileHandle file_out; SmartPtr<DrmBoBuffer> input_buf, output_buf; VideoBufferInfo input_buf_info, output_buf_info; SmartPtr<CLImage360Stitch> image_360; @@ -126,12 +136,16 @@ int main (int argc, char *argv[]) #if HAVE_OPENCV bool fm_ocl = false; #endif + int fisheye_num = 2; + bool all_in_one = true; bool need_save_output = true; double framerate = 30.0; - const char *file_in_name = NULL; + const char *file_in_name[XCAM_STITCH_FISHEYE_MAX_NUM] = {NULL}; const char *file_out_name = NULL; + int input_count = 0; + const struct option long_opts[] = { {"input", required_argument, NULL, 'i'}, {"output", required_argument, NULL, 'o'}, @@ -147,6 +161,8 @@ int main (int argc, char *argv[]) #if HAVE_OPENCV {"fm-ocl", required_argument, NULL, 'O'}, #endif + {"fisheye-num", required_argument, NULL, 'N'}, + {"all-in-one", required_argument, NULL, 'A'}, {"save", required_argument, NULL, 's'}, {"framerate", required_argument, NULL, 'f'}, {"loop", required_argument, NULL, 'l'}, @@ -159,7 +175,8 @@ int main (int argc, char *argv[]) switch (opt) { case 'i': XCAM_ASSERT (optarg); - file_in_name = optarg; + file_in_name[input_count] = optarg; + input_count++; break; case 'o': XCAM_ASSERT (optarg); @@ -180,6 +197,8 @@ int main (int argc, char *argv[]) case 'R': if (!strcasecmp (optarg, "1080p")) res_mode = StitchRes1080P; + else if (!strcasecmp (optarg, "1080p4")) + res_mode = StitchRes1080P4; else if (!strcasecmp (optarg, "4k")) res_mode = StitchRes4K; else { @@ -211,6 +230,16 @@ int main (int argc, char *argv[]) fm_ocl = (strcasecmp (optarg, "true") == 0 ? true : false); break; #endif + case 'N': + fisheye_num = atoi(optarg); + if (fisheye_num > XCAM_STITCH_FISHEYE_MAX_NUM) { + XCAM_LOG_ERROR ("fisheye number should not be greater than %d\n", XCAM_STITCH_FISHEYE_MAX_NUM); + return -1; + } + break; + case 'A': + all_in_one = (strcasecmp (optarg, "false") == 0 ? false : true); + break; case 's': need_save_output = (strcasecmp (optarg, "false") == 0 ? false : true); break; @@ -236,18 +265,31 @@ int main (int argc, char *argv[]) return -1; } - if (!file_in_name || !file_out_name) { - XCAM_LOG_ERROR ("input/output path is NULL"); + if (!all_in_one && input_count != fisheye_num) { + XCAM_LOG_ERROR ("multiple-input mode: conflicting input number(%d) and fisheye number(%d)", + input_count, fisheye_num); return -1; } - output_width = XCAM_ALIGN_UP (output_width, XCAM_ALIGNED_WIDTH); - output_height = XCAM_ALIGN_UP (output_height, XCAM_ALIGNED_WIDTH); - if (output_width != output_height * 2) { - XCAM_LOG_ERROR ("incorrect output size width:%d height:%d", output_width, output_height); + for (int i = 0; i < input_count; i++) { + if (!file_in_name[i]) { + XCAM_LOG_ERROR ("input[%d] path is NULL", i); + return -1; + } + } + + if (!file_out_name) { + XCAM_LOG_ERROR ("output path is NULL"); return -1; } + output_width = XCAM_ALIGN_UP (output_width, XCAM_ALIGNED_WIDTH); + output_height = XCAM_ALIGN_UP (output_height, XCAM_ALIGNED_WIDTH); + // if (output_width != output_height * 2) { + // XCAM_LOG_ERROR ("incorrect output size width:%d height:%d", output_width, output_height); + // return -1; + // } + #if !HAVE_OPENCV if (need_save_output) { XCAM_LOG_WARNING ("non-OpenCV mode, can't save video"); @@ -256,13 +298,19 @@ int main (int argc, char *argv[]) #endif printf ("Description------------------------\n"); - printf ("input file:\t\t%s\n", file_in_name); + if (all_in_one) + printf ("input file:\t\t%s\n", file_in_name[0]); + else { + for (int i = 0; i < input_count; i++) + printf ("input file %d:\t\t%s\n", i, file_in_name[i]); + } printf ("output file:\t\t%s\n", file_out_name); printf ("input width:\t\t%d\n", input_width); printf ("input height:\t\t%d\n", input_height); printf ("output width:\t\t%d\n", output_width); printf ("output height:\t\t%d\n", output_height); - printf ("resolution mode:\t%s\n", res_mode == StitchRes1080P ? "1080P" : "4K"); + printf ("resolution mode:\t%s\n", + res_mode == StitchRes1080P ? "1080P" : (res_mode == StitchRes1080P4 ? "1080P4" : "4K")); printf ("scale mode:\t\t%s\n", scale_mode == CLBlenderScaleLocal ? "local" : "global"); printf ("seam mask:\t\t%s\n", enable_seam ? "true" : "false"); printf ("fisheye map:\t\t%s\n", enable_fisheye_map ? "true" : "false"); @@ -270,6 +318,8 @@ int main (int argc, char *argv[]) #if HAVE_OPENCV printf ("feature match ocl:\t%s\n", fm_ocl ? "true" : "false"); #endif + printf ("fisheye number:\t\t%d\n", fisheye_num); + printf ("all in one:\t\t%s\n", all_in_one ? "true" : "false"); printf ("save file:\t\t%s\n", need_save_output ? "true" : "false"); printf ("framerate:\t\t%.3lf\n", framerate); printf ("loop count:\t\t%d\n", loop); @@ -279,7 +329,7 @@ int main (int argc, char *argv[]) image_360 = create_image_360_stitch ( context, enable_seam, scale_mode, - enable_fisheye_map, enable_lsc, res_mode).dynamic_cast_ptr<CLImage360Stitch> (); + enable_fisheye_map, enable_lsc, res_mode, fisheye_num, all_in_one).dynamic_cast_ptr<CLImage360Stitch> (); XCAM_ASSERT (image_360.ptr ()); image_360->set_output_size (output_width, output_height); #if HAVE_OPENCV @@ -289,17 +339,21 @@ int main (int argc, char *argv[]) input_buf_info.init (input_format, input_width, input_height); output_buf_info.init (input_format, output_width, output_height); - display = DrmDisplay::instance (); - buf_pool = new DrmBoBufferPool (display); - XCAM_ASSERT (buf_pool.ptr ()); - buf_pool->set_video_info (input_buf_info); - if (!buf_pool->reserve (6)) { - XCAM_LOG_ERROR ("init buffer pool failed"); - return -1; + for (int i = 0; i < input_count; i++) { + display[i] = DrmDisplay::instance (); + buf_pool[i] = new DrmBoBufferPool (display[i]); + XCAM_ASSERT (buf_pool[i].ptr ()); + buf_pool[i]->set_video_info (input_buf_info); + if (!buf_pool[i]->reserve (6)) { + XCAM_LOG_ERROR ("init buffer pool failed"); + return -1; + } } - ret = file_in.open (file_in_name, "rb"); - CHECK (ret, "open %s failed", file_in_name); + for (int i = 0; i < input_count; i++) { + ret = file_in[i].open (file_in_name[i], "rb"); + CHECK (ret, "open %s failed", file_in_name[i]); + } #if HAVE_OPENCV init_opencv_ocl (context); @@ -314,22 +368,41 @@ int main (int argc, char *argv[]) } #endif - int i = 0; + SmartPtr<DrmBoBuffer> pre_buf, cur_buf; +#if (HAVE_OPENCV) && (XCAM_TEST_STITCH_DEBUG) + SmartPtr<DrmBoBuffer> input_bufs[XCAM_STITCH_FISHEYE_MAX_NUM]; +#endif while (loop--) { - ret = file_in.rewind (); - CHECK (ret, "image_360 stitch rewind file(%s) failed", file_in_name); + for (int i = 0; i < input_count; i++) { + ret = file_in[i].rewind (); + CHECK (ret, "image_360 stitch rewind file(%s) failed", file_in_name[i]); + } do { - input_buf = buf_pool->get_buffer (buf_pool).dynamic_cast_ptr<DrmBoBuffer> (); - XCAM_ASSERT (input_buf.ptr ()); - read_buf = input_buf; - ret = file_in.read_buf (read_buf); + for (int i = 0; i < input_count; i++) { + cur_buf = buf_pool[i]->get_buffer (buf_pool[i]).dynamic_cast_ptr<DrmBoBuffer> (); + XCAM_ASSERT (cur_buf.ptr ()); + read_buf = cur_buf; + ret = file_in[i].read_buf (read_buf); + if (ret == XCAM_RETURN_BYPASS) + break; + if (ret == XCAM_RETURN_ERROR_FILE) { + XCAM_LOG_ERROR ("read buffer from %s failed", file_in_name[i]); + return -1; + } + + if (i == 0) + input_buf = cur_buf; + else + pre_buf->attach_buffer (cur_buf); + + pre_buf = cur_buf; +#if (HAVE_OPENCV) && (XCAM_TEST_STITCH_DEBUG) + input_bufs[i] = cur_buf; +#endif + } if (ret == XCAM_RETURN_BYPASS) break; - if (ret == XCAM_RETURN_ERROR_FILE) { - XCAM_LOG_ERROR ("read buffer from %s failed", file_in_name); - return -1; - } ret = image_360->execute (input_buf, output_buf); CHECK (ret, "image_360 stitch execute failed"); @@ -338,40 +411,57 @@ int main (int argc, char *argv[]) if (need_save_output) { cv::Mat out_mat; convert_to_mat (context, output_buf, out_mat); + writer.write (out_mat); + #if XCAM_TEST_STITCH_DEBUG - StitchInfo stitch_info = image_360->get_stitch_info (); - - static int frame = 0; - char file_name [1024]; - std::snprintf (file_name, 1023, "orig_fisheye_%d.jpg", frame); - - cv::Mat in_mat; - convert_to_mat (context, input_buf, in_mat); - cv::circle (in_mat, cv::Point(stitch_info.fisheye_info[0].center_x, stitch_info.fisheye_info[0].center_y), - stitch_info.fisheye_info[0].radius, cv::Scalar(0, 0, 255), 2); - cv::circle (in_mat, cv::Point(stitch_info.fisheye_info[1].center_x, stitch_info.fisheye_info[1].center_y), - stitch_info.fisheye_info[1].radius, cv::Scalar(0, 255, 0), 2); - cv::imwrite (file_name, in_mat); - - char frame_str[1024]; - std::snprintf (frame_str, 1023, "%d", frame); - cv::putText (out_mat, frame_str, cv::Point(120, 120), cv::FONT_HERSHEY_COMPLEX, 2.0, - cv::Scalar(0, 0, 255), 2, 8, false); - std::snprintf (file_name, 1023, "stitched_img_%d.jpg", frame); - cv::imwrite (file_name, out_mat); - - frame++; + dbg_write_image (context, image_360, input_bufs, output_buf, all_in_one, fisheye_num, input_count); #endif - writer.write (out_mat); } else #endif ensure_gpu_buffer_done (output_buf); FPS_CALCULATION (image_stitching, XCAM_OBJ_DUR_FRAME_NUM); - ++i; } while (true); } return 0; } +#if (HAVE_OPENCV) && (XCAM_TEST_STITCH_DEBUG) +static void dbg_write_image ( + SmartPtr<CLContext> context, SmartPtr<CLImage360Stitch> image_360, + SmartPtr<DrmBoBuffer> input_bufs[], SmartPtr<DrmBoBuffer> output_buf, + bool all_in_one, int fisheye_num, int input_count) +{ + cv::Mat mat; + static int frame_count = 0; + char file_name [1024]; + StitchInfo stitch_info = image_360->get_stitch_info (); + + std::snprintf (file_name, 1023, "orig_fisheye_%d.jpg", frame_count); + for (int i = 0; i < input_count; i++) { + if (!all_in_one) + std::snprintf (file_name, 1023, "orig_fisheye_%d_%d.jpg", frame_count, i); + + convert_to_mat (context, input_bufs[i], mat); + int fisheye_per_frame = all_in_one ? fisheye_num : 1; + for (int i = 0; i < fisheye_per_frame; i++) { + cv::circle (mat, cv::Point(stitch_info.fisheye_info[i].center_x, stitch_info.fisheye_info[i].center_y), + stitch_info.fisheye_info[i].radius, cv::Scalar(0, 0, 255), 2); + } + cv::imwrite (file_name, mat); + } + + char frame_str[1024]; + std::snprintf (frame_str, 1023, "%d", frame_count); + + convert_to_mat (context, output_buf, mat); + cv::putText (mat, frame_str, cv::Point(120, 120), cv::FONT_HERSHEY_COMPLEX, 2.0, + cv::Scalar(0, 0, 255), 2, 8, false); + std::snprintf (file_name, 1023, "stitched_img_%d.jpg", frame_count); + cv::imwrite (file_name, mat); + + frame_count++; +} +#endif + diff --git a/wrapper/gstreamer/gstxcamfilter.h b/wrapper/gstreamer/gstxcamfilter.h index 6be7060..261d159 100644 --- a/wrapper/gstreamer/gstxcamfilter.h +++ b/wrapper/gstreamer/gstxcamfilter.h @@ -66,7 +66,7 @@ typedef enum { enum StitchResMode { StitchRes1080P = 0, - StitchRes4K + StitchRes4K = 2 }; typedef struct _GstXCamFilter GstXCamFilter; |