soft-stitcher: enable stitcher

Signed-off-by: Wind Yuan <feng.yuan@intel.com>

3 files changed, 810 insertions, 0 deletions

diff --git a/modules/soft/Makefile.am b/modules/soft/Makefile.am
index 928ad0e..fcdba34 100644
--- a/modules/soft/Makefile.am
+++ b/modules/soft/Makefile.am
@@ -13,6 +13,7 @@ xcam_soft_sources =                  \
     soft_blender_tasks_priv.cpp      \
     soft_blender.cpp                 \
     soft_geo_mapper.cpp              \
+    soft_stitcher.cpp                \
    $(NULL)
 
 libxcam_soft_la_SOURCES =   \
@@ -42,6 +43,7 @@ nobase_libxcam_softinclude_HEADERS =   \
     soft_image.h                       \
     soft_blender.h                     \
     soft_geo_mapper.h                  \
+    soft_stitcher.h                    \
     $(NULL)
 
 noinst_HEADERS =                       \
diff --git a/modules/soft/soft_stitcher.cpp b/modules/soft/soft_stitcher.cpp
new file mode 100644
index 0000000..4c59931
--- /dev/null
+++ b/modules/soft/soft_stitcher.cpp
@@ -0,0 +1,722 @@
+/*
+ * soft_stitcher.cpp - soft stitcher implementation
+ *
+ *  Copyright (c) 2017 Intel Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Author: Wind Yuan <feng.yuan@intel.com>
+ */
+
+#include "soft_stitcher.h"
+#include "soft_blender.h"
+#include "soft_geo_mapper.h"
+#include "soft_video_buf_allocator.h"
+#include "interface/feature_match.h"
+#include "surview_fisheye_dewarp.h"
+#include <map>
+
+#define SOFT_STITCHER_ALIGNMENT_X 8
+#define SOFT_STITCHER_ALIGNMENT_Y 4
+
+#define MAP_FACTOR_X  16
+#define MAP_FACTOR_Y  16
+
+#define ENABLE_FEATURE_MATCH 0
+
+#define DUMP_STITCHER 1
+
+namespace XCam {
+
+#if DUMP_STITCHER
+static void
+stitcher_dump_buf (const SmartPtr<VideoBuffer> buf, uint32_t idx, const char *prefix)
+{
+    XCAM_ASSERT (prefix);
+    char name[256];
+    snprintf (name, 256, "%s-%d", prefix, idx);
+    dump_buf_perfix_path (buf, name);
+}
+#else
+static void stitcher_dump_buf (...) {}
+#endif
+
+
+namespace SoftSitcherPriv {
+
+DECLARE_HANDLER_CALLBACK (CbGeoMap, SoftStitcher, dewarp_done);
+DECLARE_HANDLER_CALLBACK (CbBlender, SoftStitcher, blender_done);
+
+struct BlenderParam
+    : SoftBlender::BlenderParam
+{
+    SmartPtr<SoftStitcher::StitcherParam>  stitch_param;
+    uint32_t idx;
+
+    BlenderParam (
+        uint32_t i,
+        const SmartPtr<VideoBuffer> &in0,
+        const SmartPtr<VideoBuffer> &in1,
+        const SmartPtr<VideoBuffer> &out)
+        : SoftBlender::BlenderParam (in0, in1, out)
+        , idx (i)
+    {}
+};
+
+typedef std::map<void*, SmartPtr<BlenderParam>> BlenderParams;
+typedef std::map<void*, int32_t> BlendCopyTaskNums;
+
+struct HandlerParam
+    : ImageHandler::Parameters
+{
+    SmartPtr<SoftStitcher::StitcherParam>  stitch_param;
+    uint32_t idx;
+
+    HandlerParam (uint32_t i)
+        : idx (i)
+    {}
+};
+
+struct Factor {
+    float x, y;
+
+    Factor () : x (1.0f), y (1.0f) {}
+    void reset () {
+        x = 1.0f;
+        y = 1.0f;
+    }
+};
+
+struct Overlap {
+    SmartPtr<FeatureMatch>       matcher;
+    SmartPtr<SoftBlender>        blender;
+    BlenderParams                param_map;
+
+    SmartPtr<BlenderParam> find_blender_param_in_map (
+        const SmartPtr<SoftStitcher::StitcherParam> &key,
+        const uint32_t idx);
+};
+
+struct FisheyeDewarp {
+    SmartPtr<SoftGeoMapper>      dewarp;
+    SmartPtr<BufferPool>         buf_pool;
+    Factor                       left_factor, right_factor;
+
+    bool set_dewarp_factor ();
+    XCamReturn set_dewarp_geo_table (
+        SmartPtr<SoftGeoMapper> mapper,
+        const CameraInfo &cam_info, const BowlDataConfig &bowl);
+};
+
+class StitcherImpl {
+    friend class XCam::SoftStitcher;
+
+public:
+    StitcherImpl (SoftStitcher *handler)
+        : _stitcher (handler)
+    {}
+
+    XCamReturn init_config (uint32_t count);
+
+    bool remove_task_count (const SmartPtr<SoftStitcher::StitcherParam> &param);
+    int32_t dec_task_count (const SmartPtr<SoftStitcher::StitcherParam> &param);
+
+    XCamReturn start_dewarp_works (const SmartPtr<SoftStitcher::StitcherParam> &param);
+    XCamReturn start_task_count (const SmartPtr<SoftStitcher::StitcherParam> &param);
+    XCamReturn start_overlap_tasks (
+        const SmartPtr<SoftStitcher::StitcherParam> &param,
+        const uint32_t idx, const SmartPtr<VideoBuffer> &buf);
+    XCamReturn start_copy_tasks (
+        const SmartPtr<SoftStitcher::StitcherParam> &param,
+        const uint32_t idx, const SmartPtr<VideoBuffer> &buf);
+
+    XCamReturn start_single_blender (const uint32_t idx, const SmartPtr<BlenderParam> &param);
+
+    XCamReturn fisheye_dewarp_to_table ();
+    XCamReturn feature_match (
+        const SmartPtr<VideoBuffer> &left_buf,
+        const SmartPtr<VideoBuffer> &right_buf,
+        const uint32_t idx);
+
+private:
+    XCamReturn init_fisheye (uint32_t idx);
+
+private:
+    FisheyeDewarp           _fisheye [XCAM_STITCH_MAX_CAMERAS];
+    Overlap                 _overlaps [XCAM_STITCH_MAX_CAMERAS];
+    SmartPtr<BufferPool>    _dewarp_pool;
+
+    Mutex                   _map_mutex;
+    BlendCopyTaskNums       _task_counts;
+
+    SoftStitcher           *_stitcher;
+};
+
+bool
+FisheyeDewarp::set_dewarp_factor ()
+{
+    XCAM_FAIL_RETURN (
+        ERROR, dewarp.ptr (), false,
+        "FisheyeDewarp dewarp handler empty");
+
+    Factor cur_left_factor, cur_right_factor;
+    Factor unify_factor;
+    dewarp->get_factors (unify_factor.x, unify_factor.y);
+    cur_left_factor = cur_right_factor = unify_factor;
+    if (XCAM_DOUBLE_EQUAL_AROUND(cur_left_factor.x, 0.0f) ||
+            XCAM_DOUBLE_EQUAL_AROUND(cur_right_factor.x, 0.0f)) { // not started.
+        return true;
+    }
+
+    cur_left_factor.x *= left_factor.x;
+    cur_left_factor.y *= left_factor.y;
+    cur_right_factor.x *= right_factor.x;
+    cur_right_factor.y *= right_factor.y;
+    unify_factor.x = (cur_left_factor.x + cur_right_factor.x) / 2.0f;
+    unify_factor.y = (cur_left_factor.y + cur_right_factor.y) / 2.0f;
+    dewarp->set_factors (unify_factor.x, unify_factor.y);
+
+    left_factor.reset ();
+    right_factor.reset ();
+    return true;
+}
+
+XCamReturn
+FisheyeDewarp::set_dewarp_geo_table (SmartPtr<SoftGeoMapper> mapper, const CameraInfo &cam_info, const BowlDataConfig &bowl)
+{
+    PolyFisheyeDewarp fd;
+    fd.set_intrinsic_param (cam_info.calibration.intrinsic);
+    fd.set_extrinsic_param (cam_info.calibration.extrinsic);
+
+    uint32_t table_width, table_height;
+    table_width = cam_info.slice_view.width / MAP_FACTOR_X;
+    table_width = XCAM_ALIGN_UP (table_width, 4);
+    table_height = cam_info.slice_view.height / MAP_FACTOR_Y;
+    table_height = XCAM_ALIGN_UP (table_height, 2);
+    SurViewFisheyeDewarp::MapTable map_table(table_width * table_height * 2);
+    fd.fisheye_dewarp (
+        map_table, table_width, table_height,
+        cam_info.slice_view.width, cam_info.slice_view.height, bowl);
+
+    std::vector<GeoData> geo_data (table_width * table_height);
+
+    for (uint32_t row = 0; row < table_height; row++) {
+        for(uint32_t col = 0; col < table_width; col++) {
+            GeoData &data = geo_data[row * table_width + col];
+            data.x = map_table[row * table_width * 2 + col * 2];
+            data.y = map_table[row * table_width * 2 + col * 2 + 1];
+            data.z = 0.0f;
+            data.w = 0.0f;
+        }
+    }
+
+    XCAM_FAIL_RETURN (
+        ERROR, mapper->set_lookup_table (geo_data.data (), table_width, table_height),
+        XCAM_RETURN_ERROR_UNKNOWN, "set fisheye dewarp lookup table failed");
+    return XCAM_RETURN_NO_ERROR;
+}
+
+XCamReturn
+StitcherImpl::init_fisheye (uint32_t idx)
+{
+    FisheyeDewarp &fisheye = _fisheye[idx];
+    SmartPtr<ImageHandler::Callback> dewarp_cb = new CbGeoMap (_stitcher);
+    fisheye.dewarp = new SoftGeoMapper ("sitcher_remapper");
+    XCAM_ASSERT (fisheye.dewarp.ptr ());
+    fisheye.dewarp->set_callback (dewarp_cb);
+
+    CameraInfo cam_info;
+    _stitcher->get_camera_info (idx, cam_info);
+
+    VideoBufferInfo buf_info;
+    buf_info.init (
+        V4L2_PIX_FMT_NV12, cam_info.slice_view.width, cam_info.slice_view.height,
+        XCAM_ALIGN_UP (cam_info.slice_view.width, SOFT_STITCHER_ALIGNMENT_X),
+        XCAM_ALIGN_UP (cam_info.slice_view.height, SOFT_STITCHER_ALIGNMENT_Y));
+
+    fisheye.buf_pool = new SoftVideoBufAllocator (buf_info);
+    XCAM_ASSERT (fisheye.buf_pool.ptr ());
+    XCAM_FAIL_RETURN (
+        ERROR, fisheye.buf_pool->reserve (2), XCAM_RETURN_ERROR_MEM,
+        "stitcher:%s reserve dewarp buffer pool(w:%d,h:%d) failed",
+        XCAM_STR (_stitcher->get_name ()), buf_info.width, buf_info.height);
+    return XCAM_RETURN_NO_ERROR;
+}
+
+XCamReturn
+StitcherImpl::init_config (uint32_t count)
+{
+    SmartPtr<ImageHandler::Callback> blender_cb = new CbBlender (_stitcher);
+    for (uint32_t i = 0; i < count; ++i) {
+        XCamReturn ret = init_fisheye (i);
+        XCAM_FAIL_RETURN (
+            ERROR, xcam_ret_is_ok (ret), ret,
+            "stitcher:%s init fisheye failed, idx:%d.", XCAM_STR (_stitcher->get_name ()), i);
+
+#if ENABLE_FEATURE_MATCH
+        _overlaps[i].matcher = new CVCapiFeatureMatch;
+        //_overlaps[i].matcher->set_config ();
+        _overlaps[i].matcher->set_fm_index (i);
+#endif
+
+        _overlaps[i].blender = create_soft_blender ().dynamic_cast_ptr<SoftBlender>();
+        XCAM_ASSERT (_overlaps[i].blender.ptr ());
+        _overlaps[i].blender->set_callback (blender_cb);
+        _overlaps[i].param_map.clear ();
+    }
+
+    return XCAM_RETURN_NO_ERROR;
+}
+
+bool
+StitcherImpl::remove_task_count (const SmartPtr<SoftStitcher::StitcherParam> &param)
+{
+    XCAM_ASSERT (param.ptr ());
+    SmartLock locker (_map_mutex);
+    BlendCopyTaskNums::iterator i = _task_counts.find (param.ptr ());
+    if (i == _task_counts.end ())
+        return false;
+
+    _task_counts.erase (i);
+    return true;
+}
+
+int32_t
+StitcherImpl::dec_task_count (const SmartPtr<SoftStitcher::StitcherParam> &param)
+{
+    XCAM_ASSERT (param.ptr ());
+    SmartLock locker (_map_mutex);
+    BlendCopyTaskNums::iterator i = _task_counts.find (param.ptr ());
+    if (i == _task_counts.end ())
+        return -1;
+
+    int32_t &count = i->second;
+    --count;
+    if (count > 0)
+        return count;
+
+    XCAM_ASSERT (count == 0);
+    _task_counts.erase (i);
+    return 0;
+}
+
+XCamReturn
+StitcherImpl::fisheye_dewarp_to_table ()
+{
+    std::vector<GeoData> table;
+    uint32_t camera_num = _stitcher->get_camera_num ();
+    for (uint32_t i = 0; i < camera_num; ++i) {
+        CameraInfo cam_info;
+        _stitcher->get_camera_info (i, cam_info);
+
+        BowlDataConfig bowl = _stitcher->get_bowl_config ();
+        bowl.angle_start = cam_info.slice_view.hori_angle_start;
+        bowl.angle_end = format_angle (
+                             cam_info.slice_view.hori_angle_start + cam_info.slice_view.hori_angle_range);
+
+        uint32_t out_width, out_height;
+        _stitcher->get_output_size (out_width, out_height);
+        XCAM_FAIL_RETURN (
+            ERROR, bowl.ground_image_height + bowl.wall_image_height == out_height, XCAM_RETURN_ERROR_PARAM,
+            "stitcher:%s bowl height(ground_height:%d + wall_height%d) not equal to out_height:%d failed",
+            XCAM_STR (_stitcher->get_name ()),
+            bowl.ground_image_height, bowl.wall_image_height, out_height);
+
+        _fisheye[i].dewarp->set_output_size (cam_info.slice_view.width, cam_info.slice_view.height);
+        //FIXME,  Fisheyedewarp need reverse angle
+        float tmp = bowl.angle_start;
+        bowl.angle_start = bowl.angle_end;
+        bowl.angle_end = tmp;
+        if (bowl.angle_end > bowl.angle_start)
+            bowl.angle_end -= 360.0f;
+        //
+        XCAM_LOG_INFO (
+            "soft-stitcher:%s camera(idx:%d) info (angle start:%.2f, range:%.2f), bowl info (angle start%.2f, end:%.2f)",
+            XCAM_STR (_stitcher->get_name ()), i,
+            cam_info.slice_view.hori_angle_start, cam_info.slice_view.hori_angle_range,
+            bowl.angle_start, bowl.angle_end);
+        XCamReturn ret = _fisheye[i].set_dewarp_geo_table (_fisheye[i].dewarp, cam_info, bowl);
+        XCAM_FAIL_RETURN (
+            ERROR, xcam_ret_is_ok (ret), ret,
+            "stitcher:%s set dewarp geo table failed, idx:%d.", XCAM_STR (_stitcher->get_name ()), i);
+
+    }
+
+    return XCAM_RETURN_NO_ERROR;
+}
+
+XCamReturn
+StitcherImpl::start_dewarp_works (const SmartPtr<SoftStitcher::StitcherParam> &param)
+{
+    uint32_t camera_num = _stitcher->get_camera_num ();
+    for (uint32_t i = 0; i < camera_num; ++i) {
+        SmartPtr<VideoBuffer> out_buf = _fisheye[i].buf_pool->get_buffer ();
+        SmartPtr<HandlerParam> dewarp_params = new HandlerParam (i);
+        dewarp_params->in_buf = param->in_bufs[i];
+        dewarp_params->out_buf = out_buf;
+        dewarp_params->stitch_param = param;
+        _fisheye[i].set_dewarp_factor ();
+        XCamReturn ret = _fisheye[i].dewarp->execute_buffer (dewarp_params, false);
+        XCAM_FAIL_RETURN (
+            ERROR, xcam_ret_is_ok (ret), ret,
+            "soft-stitcher:%s fisheye dewarp buffer failed", XCAM_STR (_stitcher->get_name ()));
+    }
+    return XCAM_RETURN_NO_ERROR;
+}
+
+SmartPtr<BlenderParam>
+Overlap::find_blender_param_in_map (
+    const SmartPtr<SoftStitcher::StitcherParam> &key,
+    const uint32_t idx)
+{
+    SmartPtr<BlenderParam> param;
+    BlenderParams::iterator i = param_map.find (key.ptr ());
+    if (i == param_map.end ()) {
+        param = new BlenderParam (idx, NULL, NULL, NULL);
+        XCAM_ASSERT (param.ptr ());
+        param->stitch_param = key;
+        param_map.insert (std::make_pair ((void*)key.ptr (), param));
+    } else {
+        param = (*i).second;
+    }
+
+    return param;
+}
+
+XCamReturn
+StitcherImpl::feature_match (
+    const SmartPtr<VideoBuffer> &left_buf,
+    const SmartPtr<VideoBuffer> &right_buf,
+    const uint32_t idx)
+{
+    const ImageOverlapInfo overlap_info = _stitcher->get_overlap (idx);
+    Rect left_ovlap = overlap_info.left;
+    Rect right_ovlap = overlap_info.right;
+    const VideoBufferInfo left_buf_info = left_buf->get_video_info ();
+
+    _overlaps[idx].matcher->reset_offsets ();
+    _overlaps[idx].matcher->optical_flow_feature_match (
+        left_buf, right_buf, left_ovlap, right_ovlap, left_buf_info.width);
+    float left_offsetx = _overlaps[idx].matcher->get_current_left_offset_x ();
+
+    uint32_t left_idx = idx;
+    float center_x = (float) _stitcher->get_center (left_idx).slice_center_x;
+    float feature_center_x = (float)left_ovlap.pos_x + (left_ovlap.width / 2.0f);
+    float range = feature_center_x - center_x;
+    XCAM_ASSERT (range > 1.0f);
+    _fisheye[left_idx].right_factor.x = (range + left_offsetx / 2.0f) / range;
+    _fisheye[left_idx].right_factor.y = 1.0;
+    XCAM_ASSERT (_fisheye[left_idx].right_factor.x > 0.0f && _fisheye[left_idx].right_factor.x < 2.0f);
+
+    uint32_t right_idx = (idx + 1) % _stitcher->get_camera_num ();
+    center_x = (float) _stitcher->get_center (right_idx).slice_center_x;
+    feature_center_x = (float)right_ovlap.pos_x + (right_ovlap.width / 2.0f);
+    range = center_x - feature_center_x;
+    XCAM_ASSERT (range > 1.0f);
+    _fisheye[right_idx].left_factor.x = (range + left_offsetx / 2.0f) / range;
+    _fisheye[right_idx].left_factor.y = 1.0;
+    XCAM_ASSERT (_fisheye[right_idx].left_factor.x > 0.0f && _fisheye[right_idx].left_factor.x < 2.0f);
+
+    return XCAM_RETURN_NO_ERROR;
+}
+
+XCamReturn
+StitcherImpl::start_single_blender (
+    const uint32_t idx,
+    const SmartPtr<BlenderParam> &param)
+{
+    SmartPtr<SoftBlender> blender = _overlaps[idx].blender;
+    const ImageOverlapInfo &overlap_info = _stitcher->get_overlap (idx);
+    uint32_t out_width, out_height;
+    _stitcher->get_output_size (out_width, out_height);
+
+    blender->set_output_size (out_width, out_height);
+    blender->set_input_valid_area (overlap_info.left, 0);
+    blender->set_input_valid_area (overlap_info.right, 1);
+    blender->set_input_merge_area (overlap_info.left, 0);
+    blender->set_input_merge_area (overlap_info.right, 1);
+    blender->set_merge_window (overlap_info.out_area);
+    return blender->execute_buffer (param, false);
+}
+
+XCamReturn
+StitcherImpl::start_overlap_tasks (
+    const SmartPtr<SoftStitcher::StitcherParam> &param,
+    const uint32_t idx, const SmartPtr<VideoBuffer> &buf)
+{
+    SmartPtr<BlenderParam> cur_param, prev_param;
+    const uint32_t camera_num = _stitcher->get_camera_num ();
+    uint32_t pre_idx = (idx + camera_num - 1) % camera_num;
+    XCamReturn ret = XCAM_RETURN_NO_ERROR;
+    {
+        SmartPtr<BlenderParam> param_b;
+
+        SmartLock locker (_map_mutex);
+        param_b = _overlaps[idx].find_blender_param_in_map (param, idx);
+        param_b->in_buf = buf;
+        if (param_b->in_buf.ptr () && param_b->in1_buf.ptr ()) {
+            cur_param = param_b;
+            _overlaps[idx].param_map.erase (param.ptr ());
+        }
+
+        param_b = _overlaps[pre_idx].find_blender_param_in_map (param, idx);
+        param_b->in1_buf = buf;
+        if (param_b->in_buf.ptr () && param_b->in1_buf.ptr ()) {
+            prev_param = param_b;
+            _overlaps[pre_idx].param_map.erase (param.ptr ());
+        }
+    }
+
+    if (cur_param.ptr ()) {
+        cur_param->out_buf = param->out_buf;
+        ret = start_single_blender (idx, cur_param);
+        XCAM_FAIL_RETURN (
+            ERROR, xcam_ret_is_ok (ret), ret,
+            "soft-stitcher:%s blend overlap idx:%d failed", XCAM_STR (_stitcher->get_name ()), idx);
+    }
+
+    if (prev_param.ptr ()) {
+        prev_param->out_buf = param->out_buf;
+        ret = start_single_blender (pre_idx, prev_param);
+        XCAM_FAIL_RETURN (
+            ERROR, xcam_ret_is_ok (ret), ret,
+            "soft-stitcher:%s blend overlap idx:%d failed", XCAM_STR (_stitcher->get_name ()), pre_idx);
+    }
+
+#if ENABLE_FEATURE_MATCH
+    //start feature match
+    if (cur_param.ptr ()) {
+        ret = feature_match (cur_param->in_buf, cur_param->in1_buf, idx);
+        XCAM_FAIL_RETURN (
+            ERROR, xcam_ret_is_ok (ret), ret,
+            "soft-stitcher:%s feature-match overlap idx:%d failed", XCAM_STR (_stitcher->get_name ()), idx);
+    }
+
+    if (prev_param.ptr ()) {
+        ret = feature_match (prev_param->in_buf, prev_param->in1_buf, pre_idx);
+        XCAM_FAIL_RETURN (
+            ERROR, xcam_ret_is_ok (ret), ret,
+            "soft-stitcher:%s feature-match overlap idx:%d failed", XCAM_STR (_stitcher->get_name ()), pre_idx);
+    }
+#endif
+    return XCAM_RETURN_NO_ERROR;
+}
+
+XCamReturn
+StitcherImpl::start_copy_tasks (
+    const SmartPtr<SoftStitcher::StitcherParam> &param,
+    const uint32_t idx, const SmartPtr<VideoBuffer> &buf)
+{
+    //TODO add copy tasks
+    return XCAM_RETURN_NO_ERROR;
+}
+
+};
+
+SoftStitcher::SoftStitcher (const char *name)
+    : SoftHandler (name)
+    , Stitcher (SOFT_STITCHER_ALIGNMENT_X, SOFT_STITCHER_ALIGNMENT_Y)
+{
+    _impl = new SoftSitcherPriv::StitcherImpl (this);
+    XCAM_ASSERT (_impl.ptr ());
+}
+
+SoftStitcher::~SoftStitcher ()
+{
+}
+
+XCamReturn
+SoftStitcher::stitch_buffers (const VideoBufferList &in_bufs, SmartPtr<VideoBuffer> &out_buf)
+{
+    XCAM_FAIL_RETURN (
+        ERROR, !in_bufs.empty (), XCAM_RETURN_ERROR_PARAM,
+        "soft-stitcher:%s stitch buffer failed, in_bufs is empty", XCAM_STR (get_name ()));
+
+    SmartPtr<StitcherParam> param = new StitcherParam;
+    param->out_buf = out_buf;
+    uint32_t count = 0;
+    for (VideoBufferList::const_iterator i = in_bufs.begin(); i != in_bufs.end (); ++i) {
+        SmartPtr<VideoBuffer> buf = *i;
+        XCAM_ASSERT (buf.ptr ());
+        param->in_bufs[count++] = buf;
+    }
+    param->in_buf_num = count;
+    XCamReturn ret = execute_buffer (param, true);
+    if (!out_buf.ptr () && xcam_ret_is_ok (ret)) {
+        out_buf = param->out_buf;
+    }
+    return ret;
+}
+
+XCamReturn
+SoftStitcher::start_task_count (const SmartPtr<SoftStitcher::StitcherParam> &param)
+{
+    XCAM_ASSERT (param.ptr ());
+    XCAM_ASSERT (_impl.ptr ());
+
+    SmartLock locker (_impl->_map_mutex);
+
+    XCAM_FAIL_RETURN (
+        ERROR, check_work_continue (param, XCAM_RETURN_NO_ERROR), XCAM_RETURN_ERROR_PARAM,
+        "soft-stitcher:%s start task count failed in work check", XCAM_STR (get_name ()));
+
+    if (_impl->_task_counts.find (param.ptr ()) != _impl->_task_counts.end ()) {
+        XCAM_LOG_ERROR ("tasks already started, this should never happen.");
+        return XCAM_RETURN_ERROR_UNKNOWN;
+    }
+
+    int32_t count = get_camera_num ();
+    //count += get_copy_area ().size ();
+
+    XCAM_LOG_DEBUG ("stitcher :%s start task count :%d", XCAM_STR(get_name ()), count);
+    _impl->_task_counts.insert (std::make_pair((void*)param.ptr(), count));
+    return XCAM_RETURN_NO_ERROR;
+}
+
+void
+SoftStitcher::dewarp_done (
+    const SmartPtr<ImageHandler> &handler,
+    const SmartPtr<ImageHandler::Parameters> &base,
+    const XCamReturn error)
+{
+    SmartPtr<SoftSitcherPriv::HandlerParam> dewarp_param = base.dynamic_cast_ptr<SoftSitcherPriv::HandlerParam> ();
+    XCAM_ASSERT (dewarp_param.ptr ());
+    SmartPtr<SoftStitcher::StitcherParam> param = dewarp_param->stitch_param;
+    XCAM_ASSERT (param.ptr ());
+    XCAM_UNUSED (handler);
+
+    if (!check_work_continue (param, error))
+        return;
+
+    stitcher_dump_buf (dewarp_param->out_buf, dewarp_param->idx, "stitcher-dewarp");
+
+    //start both blender and feature match
+    XCamReturn ret = _impl->start_overlap_tasks (param, dewarp_param->idx, dewarp_param->out_buf);
+    if (!xcam_ret_is_ok (ret)) {
+        work_broken (param, ret);
+    }
+
+    ret = _impl->start_copy_tasks (param, dewarp_param->idx, dewarp_param->out_buf);
+    if (!xcam_ret_is_ok (ret)) {
+        work_broken (param, ret);
+    }
+}
+
+void
+SoftStitcher::blender_done (
+    const SmartPtr<ImageHandler> &handler,
+    const SmartPtr<ImageHandler::Parameters> &base,
+    const XCamReturn error)
+{
+    SmartPtr<SoftSitcherPriv::BlenderParam> blender_param = base.dynamic_cast_ptr<SoftSitcherPriv::BlenderParam> ();
+    XCAM_ASSERT (blender_param.ptr ());
+    SmartPtr<SoftStitcher::StitcherParam> param = blender_param->stitch_param;
+    XCAM_ASSERT (param.ptr ());
+    XCAM_UNUSED (handler);
+
+    if (!check_work_continue (param, error)) {
+        _impl->remove_task_count (param);
+        return;
+    }
+
+    stitcher_dump_buf (blender_param->out_buf, blender_param->idx, "stitcher-blend");
+    XCAM_LOG_DEBUG ("blender:(%s) done", XCAM_STR (handler->get_name ()));
+
+    if (_impl->dec_task_count (param) == 0) {
+        work_well_done (param, error);
+    }
+}
+
+XCamReturn
+SoftStitcher::configure_resource (const SmartPtr<Parameters> &param)
+{
+    XCAM_UNUSED (param);
+    XCAM_ASSERT (_impl.ptr ());
+
+    uint32_t camera_count = get_camera_num ();
+    XCamReturn ret = _impl->init_config (camera_count);
+    XCAM_FAIL_RETURN (
+        ERROR, xcam_ret_is_ok (ret), ret,
+        "soft-stitcher:%s initialize private config failed", XCAM_STR (get_name ()));
+
+    ret = estimate_coarse_crops ();
+    XCAM_FAIL_RETURN (
+        ERROR, xcam_ret_is_ok (ret), ret,
+        "soft-stitcher:%s estimate coarse crops failed", XCAM_STR (get_name ()));
+
+    ret = mark_centers ();
+    XCAM_FAIL_RETURN (
+        ERROR, xcam_ret_is_ok (ret), ret,
+        "soft-stitcher:%s mark centers failed", XCAM_STR (get_name ()));
+
+    ret = estimate_overlap ();
+    XCAM_FAIL_RETURN (
+        ERROR, xcam_ret_is_ok (ret), ret,
+        "soft-stitcher:%s estimake coarse overlap failed", XCAM_STR (get_name ()));
+
+    ret = _impl->fisheye_dewarp_to_table ();
+    XCAM_FAIL_RETURN (
+        ERROR, xcam_ret_is_ok (ret), ret,
+        "soft-stitcher:%s fisheye_dewarp_to_table failed", XCAM_STR (get_name ()));
+
+    VideoBufferInfo out_info;
+    uint32_t out_width, out_height;
+    get_output_size (out_width, out_height);
+    XCAM_FAIL_RETURN (
+        ERROR, out_width && out_height, XCAM_RETURN_ERROR_PARAM,
+        "soft-stitcher:%s output size was not set", XCAM_STR(get_name ()));
+
+    out_info.init (
+        V4L2_PIX_FMT_NV12, out_width, out_height,
+        XCAM_ALIGN_UP (out_width, SOFT_STITCHER_ALIGNMENT_X),
+        XCAM_ALIGN_UP (out_height, SOFT_STITCHER_ALIGNMENT_Y));
+    set_out_video_info (out_info);
+
+    return ret;
+}
+
+XCamReturn
+SoftStitcher::start_work (const SmartPtr<Parameters> &base)
+{
+    SmartPtr<StitcherParam> param = base.dynamic_cast_ptr<StitcherParam> ();
+
+    XCAM_FAIL_RETURN (
+        ERROR, param.ptr () && param->in_buf_num > 0 && param->in_bufs[0].ptr (), XCAM_RETURN_ERROR_PARAM,
+        "soft_stitcher:%s start_work failed, params(in_buf_num) in_bufs are set",
+        XCAM_STR (get_name ()));
+
+    XCamReturn ret = start_task_count (param);
+    XCAM_FAIL_RETURN (
+        ERROR, xcam_ret_is_ok (ret), XCAM_RETURN_ERROR_PARAM,
+        "soft_stitcher:%s start blender count failed", XCAM_STR (get_name ()));
+
+    ret = _impl->start_dewarp_works (param);
+    XCAM_FAIL_RETURN (
+        ERROR, xcam_ret_is_ok (ret), XCAM_RETURN_ERROR_PARAM,
+        "soft_stitcher:%s start dewarp works failed", XCAM_STR (get_name ()));
+
+    for (uint32_t i = 0; i < param->in_buf_num; ++i) {
+        param->in_bufs[i].release ();
+    }
+
+    return ret;
+}
+
+SmartPtr<Stitcher>
+Stitcher::create_soft_stitcher ()
+{
+    return new SoftStitcher;
+}
+
+}
+
diff --git a/modules/soft/soft_stitcher.h b/modules/soft/soft_stitcher.h
new file mode 100644
index 0000000..da62937
--- /dev/null
+++ b/modules/soft/soft_stitcher.h
@@ -0,0 +1,86 @@
+/*
+ * soft_stitcher.h - soft stitcher class
+ *
+ *  Copyright (c) 2017 Intel Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Author: Wind Yuan <feng.yuan@intel.com>
+ */
+
+#ifndef XCAM_SOFT_STITCHER_H
+#define XCAM_SOFT_STITCHER_H
+
+#include "xcam_utils.h"
+#include "interface/stitcher.h"
+#include "soft_handler.h"
+
+namespace XCam {
+
+namespace SoftSitcherPriv {
+class StitcherImpl;
+class CbGeoMap;
+class CbBlender;
+};
+
+class SoftStitcher
+    : public SoftHandler
+    , public Stitcher
+{
+    friend class SoftSitcherPriv::StitcherImpl;
+    friend class SoftSitcherPriv::CbGeoMap;
+    friend class SoftSitcherPriv::CbBlender;
+
+public:
+    struct StitcherParam
+        : ImageHandler::Parameters
+    {
+        uint32_t in_buf_num;
+        SmartPtr<VideoBuffer> in_bufs[XCAM_STITCH_MAX_CAMERAS];
+
+        StitcherParam ()
+            : Parameters (NULL, NULL)
+            , in_buf_num (0)
+        {}
+    };
+
+public:
+    explicit SoftStitcher (const char *name = "SoftStitcher");
+    ~SoftStitcher ();
+
+protected:
+    // interface derive from Stitcher
+    XCamReturn stitch_buffers (const VideoBufferList &in_bufs, SmartPtr<VideoBuffer> &out_buf);
+
+    //derived from SoftHandler
+    XCamReturn configure_resource (const SmartPtr<Parameters> &param);
+    XCamReturn start_work (const SmartPtr<Parameters> &param);
+
+private:
+    // handler done, call back functions
+    XCamReturn start_task_count (
+        const SmartPtr<SoftStitcher::StitcherParam> &param);
+    void dewarp_done (
+        const SmartPtr<ImageHandler> &handler,
+        const SmartPtr<ImageHandler::Parameters> &param, const XCamReturn error);
+    void blender_done (
+        const SmartPtr<ImageHandler> &handler,
+        const SmartPtr<ImageHandler::Parameters> &param, const XCamReturn error);
+
+private:
+    SmartPtr<SoftSitcherPriv::StitcherImpl> _impl;
+};
+
+}
+
+#endif //XCAM_SOFT_STITCHER_H