/* * dnn_inference_engine.h - dnn inference engine * * Copyright (c) 2019 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * * Author: Zong Wei */ #ifndef _XCMA_DNN_INFERENCE_ENGINE_H_ #define _XCMA_DNN_INFERENCE_ENGINE_H_ #pragma once #include #include #include #include #include #include #include namespace XCam { enum DnnInferTargetDeviceType { DnnInferDeviceDefault = 0, DnnInferDeviceBalanced = 1, DnnInferDeviceCPU = 2, DnnInferDeviceGPU = 3, DnnInferDeviceFPGA = 4, DnnInferDeviceMyriad = 5, DnnInferDeviceHetero = 8 }; enum DnnInferPrecisionType { DnnInferPrecisionMixed = 0, DnnInferPrecisionFP32 = 10, DnnInferPrecisionFP16 = 11, DnnInferPrecisionQ78 = 20, DnnInferPrecisionI16 = 30, DnnInferPrecisionU8 = 40, DnnInferPrecisionI8 = 50, DnnInferPrecisionU16 = 60, DnnInferPrecisionI32 = 70, DnnInferPrecisionCustom = 80, DnnInferPrecisionUnspecified = 255 }; enum DnnInferLayoutType { DnnInferLayoutAny = 0, DnnInferLayoutNCHW = 1, DnnInferLayoutNHWC = 2, DnnInferLayoutOIHW = 64, DnnInferLayoutC = 96, DnnInferLayoutCHW = 128, DnnInferLayoutHW = 192, DnnInferLayoutNC = 193, DnnInferLayoutCN = 194, DnnInferLayoutBlocked = 200 }; enum DnnInferMemoryType { DnnInferMemoryDefault = 0, DnnInferMemoryHost = 1, DnnInferMemoryGPU = 2, DnnInferMemoryMYRIAD = 3, DnnInferMemoryShared = 4 }; enum DnnInferImageFormatType { DnnInferImageFormatBGRPacked = 0, DnnInferImageFormatBGRPlanar, DnnInferImageFormatRGBPacked, DnnInferImageFormatRGBPlanar, DnnInferImageFormatGrayPlanar, DnnInferImageFormatGeneric1D, DnnInferImageFormatGeneric2D, DnnInferImageFormatUnknown = -1 }; enum DnnInferMode { DnnInferModeSync = 0, DnnInferModeAsync = 1 }; enum DnnInferDataType { DnnInferDataTypeNonImage = 0, DnnInferDataTypeImage = 1 }; enum DnnInferLogLevel { DnnInferLogLevelNone = 0x0, DnnInferLogLevelEngine = 0x1, DnnInferLogLevelLayer = 0x2 }; enum DnnInferInfoType { DnnInferInfoEngine = 0x0, DnnInferInfoPlugin = 0x1, DnnInferInfoNetwork = 0x2 }; struct DnnInferImageSize { uint32_t image_width; uint32_t image_height; DnnInferImageSize () { image_width = 0; image_height = 0; } }; struct DnnInferenceEngineInfo { DnnInferInfoType type; int32_t major; int32_t minor; const char* desc; const char* name; DnnInferenceEngineInfo () { type = DnnInferInfoEngine; major = 0; minor = 0; desc = NULL; name = NULL; }; }; #define DNN_INFER_MAX_INPUT_OUTPUT 10 struct DnnInferInputOutputInfo { uint32_t width[DNN_INFER_MAX_INPUT_OUTPUT]; uint32_t height[DNN_INFER_MAX_INPUT_OUTPUT]; uint32_t channels[DNN_INFER_MAX_INPUT_OUTPUT]; uint32_t object_size[DNN_INFER_MAX_INPUT_OUTPUT]; DnnInferPrecisionType precision[DNN_INFER_MAX_INPUT_OUTPUT]; DnnInferLayoutType layout[DNN_INFER_MAX_INPUT_OUTPUT]; DnnInferDataType data_type[DNN_INFER_MAX_INPUT_OUTPUT]; uint32_t batch_size; uint32_t numbers; }; struct DnnInferData { void * buffer; uint32_t size; uint32_t width; uint32_t height; uint32_t width_stride; uint32_t height_stride; uint32_t channel_num; uint32_t batch_idx; DnnInferPrecisionType precision; DnnInferMemoryType mem_type; DnnInferImageFormatType image_format; DnnInferDataType data_type; DnnInferData () { buffer = NULL; }; }; struct DnnInferConfig { DnnInferTargetDeviceType target_id; DnnInferInputOutputInfo input_infos; DnnInferInputOutputInfo output_infos; char * plugin_path; char * cpu_ext_path; char * cldnn_ext_path; char * model_filename; char * output_layer_name; uint32_t perf_counter; uint32_t infer_req_num; DnnInferConfig () { plugin_path = NULL; cpu_ext_path = NULL; cldnn_ext_path = NULL; model_filename = NULL; output_layer_name = NULL; }; }; class DnnInferenceEngine { public: explicit DnnInferenceEngine (DnnInferConfig& config); virtual ~DnnInferenceEngine (); XCamReturn create_model (DnnInferConfig& config); XCamReturn load_model (DnnInferConfig& config); XCamReturn get_info (DnnInferenceEngineInfo& info, DnnInferInfoType type); XCamReturn set_batch_size (const size_t size); size_t get_batch_size (); bool ready_to_start () const { return _model_created && _model_loaded; }; XCamReturn start (bool sync = true); size_t get_input_size (); size_t get_output_size (); XCamReturn set_input_presion (uint32_t idx, DnnInferPrecisionType precision); DnnInferPrecisionType get_input_presion (uint32_t idx); XCamReturn set_output_presion (uint32_t idx, DnnInferPrecisionType precision); DnnInferPrecisionType get_output_presion (uint32_t idx); XCamReturn set_input_layout (uint32_t idx, DnnInferLayoutType layout); XCamReturn set_output_layout (uint32_t idx, DnnInferLayoutType layout); uint32_t get_input_image_height () const { return _input_image_height; }; uint32_t get_input_image_width () const { return _input_image_width; }; XCamReturn set_model_input_info (DnnInferInputOutputInfo& info); XCamReturn get_model_input_info (DnnInferInputOutputInfo& info); XCamReturn set_model_output_info (DnnInferInputOutputInfo& info); XCamReturn get_model_output_info (DnnInferInputOutputInfo& info); XCamReturn set_inference_data (std::vector images); void* get_inference_results (uint32_t idx, uint32_t& size); std::shared_ptr read_inference_image (std::string image); void print_log (uint32_t flag); protected: InferenceEngine::TargetDevice get_device_from_string (const std::string& device_name); InferenceEngine::TargetDevice get_device_from_id (DnnInferTargetDeviceType device); InferenceEngine::Layout estimate_layout_type (const int ch_num); InferenceEngine::Layout convert_layout_type (DnnInferLayoutType layout); DnnInferLayoutType convert_layout_type (InferenceEngine::Layout layout); InferenceEngine::Precision convert_precision_type (DnnInferPrecisionType precision); DnnInferPrecisionType convert_precision_type (InferenceEngine::Precision precision); std::string get_filename_prefix (const std::string &file_path); void print_performance_counts (const std::map& performance_map); private: XCamReturn set_input_blob (uint32_t idx, DnnInferData& data); template XCamReturn copy_image_to_blob (const DnnInferData& data, InferenceEngine::Blob::Ptr& blob, int batch_index); template XCamReturn copy_data_to_blob (const DnnInferData& data, InferenceEngine::Blob::Ptr& blob, int batch_index); protected: bool _model_created; bool _model_loaded; InferenceEngine::InferencePlugin _plugin; InferenceEngine::InputsDataMap _inputs_info; InferenceEngine::OutputsDataMap _outputs_info; uint32_t _input_image_width; uint32_t _input_image_height; InferenceEngine::CNNNetReader _network_reader; InferenceEngine::CNNNetwork _network; InferenceEngine::InferRequest _infer_request; std::vector _layers; }; } // namespace XCam #endif //_XCMA_DNN_INFERENCE_ENGINE_H_