diff --git a/tools/accuracy_checker/openvino/tools/accuracy_checker/evaluators/custom_evaluators/base_models.py b/tools/accuracy_checker/openvino/tools/accuracy_checker/evaluators/custom_evaluators/base_models.py index a95e563ae50..4e0c33d0cc6 100644 --- a/tools/accuracy_checker/openvino/tools/accuracy_checker/evaluators/custom_evaluators/base_models.py +++ b/tools/accuracy_checker/openvino/tools/accuracy_checker/evaluators/custom_evaluators/base_models.py @@ -410,17 +410,58 @@ def automatic_model_search(self, network_info): class BaseOpenCVModel: def __init__(self, network_info, launcher, suffix=None, delayed_model_loading=False): - self.network_info = network_info self.launcher = launcher self.default_model_suffix = suffix if not delayed_model_loading: - self.network = launcher.create_network(network_info['model'], network_info.get('weights', '')) network_info.update(launcher.config) - input_shapes = launcher.get_inputs_from_config(network_info) - self.input_blob = next(iter(input_shapes)) - self.input_shape = input_shapes[self.input_blob] - self.network.setInputsNames(list(self.input_blob)) - self.output_blob = next(iter(self.network.getUnconnectedOutLayersNames())) + self.load_model(network_info) + + def load_model(self, network_info): + model, weights = self.automatic_model_search(network_info) + self.network = self.launcher.create_network(model, weights) + self.set_input_and_output(network_info) + + def automatic_model_search(self, network_info): + model = Path(network_info['model']) + if model.is_dir(): + is_blob = network_info.get('_model_is_blob') + if is_blob: + model_list = list(model.glob('*{}.blob'.format(self.default_model_suffix))) + if not model_list: + model_list = list(model.glob('*.blob')) + else: + model_list = list(model.glob('*{}.xml'.format(self.default_model_suffix))) + blob_list = list(model.glob('*{}.blob'.format(self.default_model_suffix))) + if not model_list and not blob_list: + model_list = list(model.glob('*.xml')) + blob_list = list(model.glob('*.blob')) + if not model_list: + model_list = blob_list + if not model_list: + raise ConfigError('Suitable model for {} not found'.format(self.default_model_suffix)) + if len(model_list) > 1: + raise ConfigError('Several suitable models for {} found'.format(self.default_model_suffix)) + model = model_list[0] + accepted_suffixes = ['.blob', '.xml', '.onnx'] + if model.suffix not in accepted_suffixes: + raise ConfigError('Models with following suffixes are allowed: {}'.format(accepted_suffixes)) + print_info('{} - Found model: {}'.format(self.default_model_suffix, model)) + if model.suffix in ['.blob', '.onnx']: + return model, None + weights = get_path(network_info.get('weights', model.parent / model.name.replace('xml', 'bin'))) + accepted_weights_suffixes = ['.bin'] + if weights.suffix not in accepted_weights_suffixes: + raise ConfigError('Weights with following suffixes are allowed: {}'.format(accepted_weights_suffixes)) + print_info('{} - Found weights: {}'.format(self.default_model_suffix, weights)) + return model, weights + + def set_input_and_output(self, network_info): + self.input_shapes = self.launcher.get_inputs_from_config(network_info) + self.input_blob = next(iter(self.input_shapes.keys())) + self.input_shape = self.input_shapes[self.input_blob] + self.network.setInputsNames(list(self.input_shapes.keys())) + self.output_names = self.network.getUnconnectedOutLayersNames() + self.output_blob = next(iter(list(self.network.getUnconnectedOutLayersNames()))) def fit_to_input(self, input_data): return {self.input_blob: input_data.astype(np.float32)} diff --git a/tools/accuracy_checker/openvino/tools/accuracy_checker/evaluators/custom_evaluators/cocosnet_evaluator.py b/tools/accuracy_checker/openvino/tools/accuracy_checker/evaluators/custom_evaluators/cocosnet_evaluator.py index 94fe5a922a0..0c71b5cde7a 100644 --- a/tools/accuracy_checker/openvino/tools/accuracy_checker/evaluators/custom_evaluators/cocosnet_evaluator.py +++ b/tools/accuracy_checker/openvino/tools/accuracy_checker/evaluators/custom_evaluators/cocosnet_evaluator.py @@ -19,7 +19,7 @@ import cv2 from .base_custom_evaluator import BaseCustomEvaluator -from .base_models import BaseDLSDKModel, BaseOpenVINOModel, BaseCascadeModel, create_model +from .base_models import BaseDLSDKModel, BaseOpenVINOModel, BaseCascadeModel, BaseOpenCVModel, create_model from ...adapters import create_adapter from ...config import ConfigError from ...data_readers import DataRepresentation @@ -119,7 +119,8 @@ def __init__(self, network_info, launcher, models_args, is_blob, delayed_model_l raise ConfigError('network_info should contain cocosnet_network field') self._test_mapping = { 'dlsdk': CocosnetModel, - 'openvino': CoCosNetModelOV + 'openvino': CoCosNetModelOV, + 'opencv': CocosnetModelOpenCV } self._check_mapping = { 'dlsdk': GanCheckModel, @@ -190,6 +191,33 @@ def predict(self, identifiers, input_data): results.append(*self.adapter.process(prediction, identifiers, [{}])) return results, prediction +class CocosnetModelOpenCV(BaseOpenCVModel): + def __init__(self, network_info, launcher, suffix=None, delayed_model_loading=False): + self.adapter = create_adapter(network_info.get('adapter')) + super().__init__(network_info, launcher, suffix, delayed_model_loading) + if self.adapter.output_blob is None: + self.adapter.output_blob = self.output_blob + self.input_names = list(self.input_shapes.keys()) + + def fit_to_input(self, input_data): + inputs = {} + for value, key in zip(input_data, self.input_names): + value = np.expand_dims(value, 0) + value = np.transpose(value, (0, 3, 1, 2)) + inputs[key] = value + return inputs + + def predict(self, identifiers, input_data): + results = [] + prediction = None + for current_input in input_data: + data = self.fit_to_input(current_input) + for input_name in data.keys(): + self.network.setInput(data[input_name].astype(np.float32), input_name) + prediction = self.network.forward(self.output_names) + dict_result = dict(zip(self.output_names, prediction)) + results.append(*self.adapter.process(dict_result, identifiers, [{}])) + return results, prediction class CoCosNetModelOV(BaseOpenVINOModel): def __init__(self, network_info, launcher, suffix=None, delayed_model_loading=False): diff --git a/tools/accuracy_checker/openvino/tools/accuracy_checker/evaluators/custom_evaluators/mtcnn_models.py b/tools/accuracy_checker/openvino/tools/accuracy_checker/evaluators/custom_evaluators/mtcnn_models.py index a5258adb0eb..5f623e67655 100644 --- a/tools/accuracy_checker/openvino/tools/accuracy_checker/evaluators/custom_evaluators/mtcnn_models.py +++ b/tools/accuracy_checker/openvino/tools/accuracy_checker/evaluators/custom_evaluators/mtcnn_models.py @@ -37,10 +37,12 @@ def build_stages(models_info, preprocessors_config, launcher, model_args, delaye stages_mapping = OrderedDict([ ('pnet', { 'caffe': CaffeProposalStage, 'dlsdk': DLSDKProposalStage, - 'dummy': DummyProposalStage, 'openvino': OpenVINOProposalStage}), + 'dummy': DummyProposalStage, 'openvino': OpenVINOProposalStage, + 'opencv': OpenCVProposalStage}), ('rnet', {'caffe': CaffeRefineStage, 'dlsdk': DLSDKRefineStage, - 'openvino': OpenVINORefineStage}), - ('onet', {'caffe': CaffeOutputStage, 'dlsdk': DLSDKOutputStage, 'openvino': OpenVINOOutputStage}) + 'openvino': OpenVINORefineStage, 'opencv': OpenCVRefineStage}), + ('onet', {'caffe': CaffeOutputStage, 'dlsdk': DLSDKOutputStage, 'openvino': OpenVINOOutputStage, + 'opencv': OpenCVOutputStage}) ]) framework = launcher.config['framework'] common_preprocessor = PreprocessingExecutor(preprocessors_config) @@ -480,6 +482,110 @@ def generate_name(prefix, with_prefix, layer_name): config_outputs[key] = generate_name(model_prefix, network_with_prefix, value) self.model_info['outputs'] = config_outputs +class OpenCVModelMixin: + def _infer(self, input_blobs, batch_meta): + for meta in batch_meta: + meta['input_shape'] = [] + results = [] + output_names = self.network.getUnconnectedOutLayersNames() + input_shapes = dict() + for feed_dict in input_blobs: + for layer_name, data in feed_dict.items(): + self.network.setInput(data.astype(np.float32), layer_name) + input_shapes.update({layer_name: data.shape}) + + list_prediction = self.network.forward(output_names) + dict_result = dict(zip(output_names, list_prediction)) + results.append(dict_result) + for meta in batch_meta: + meta['input_shape'].append(input_shapes) + + return results + + def release(self): + self.input_feeder.release() + del self.network + + def fit_to_input(self, data, layer_name, layout, precision, template=None): + layer_shape = self.inputs[layer_name] + data_shape = np.shape(data) + if len(layer_shape) == 4: + if len(data_shape) == 5: + data = data[0] + data = np.transpose(data, layout) + if precision: + data = data.astype(precision) + return data + + def prepare_model(self, model_name): + model, weights = self.auto_model_search(self.model_info, model_name) + return model, weights + + def auto_model_search(self, network_info, default_model_name): + self.default_model_name = default_model_name + model = Path(network_info.get('model', '')) + weights = network_info.get('weights') + if model.is_dir(): + models_list = list(Path(model).rglob('{}.xml'.format(self.default_model_name))) + if not models_list: + models_list = list(Path(model).glob('*.xml')) + if not models_list: + raise ConfigError('Suitable model description is not detected') + if len(models_list) != 1: + raise ConfigError('Several suitable models found, please specify required model') + model = models_list[0] + if weights is None or Path(weights).is_dir(): + weights_dir = weights or model.parent + weights = Path(weights_dir) / model.name.replace('xml', 'bin') + if not weights.exists(): + weights_list = list(weights_dir.glob('*.bin')) + if not weights_list: + raise ConfigError('Suitable weights is not detected') + if len(weights_list) != 1: + raise ConfigError('Several suitable weights found, please specify required explicitly') + weights = weights_list[0] + weights = get_path(weights) + accepted_suffixes = ['.blob', '.xml'] + if model.suffix not in accepted_suffixes: + raise ConfigError('Models with following suffixes are allowed: {}'.format(accepted_suffixes)) + print_info('{} - Found model: {}'.format(self.default_model_name, model)) + accepted_weights_suffixes = ['.bin'] + if weights.suffix not in accepted_weights_suffixes: + raise ConfigError('Weights with following suffixes are allowed: {}'.format(accepted_weights_suffixes)) + print_info('{} - Found weights: {}'.format(self.default_model_name, weights)) + return model, weights + + def load_network(self, network, launcher, model_prefix, network_info): + self.update_input_output_info(model_prefix, network_info) + self.inputs = launcher.get_inputs_from_config(network_info) + self.input_feeder = InputFeeder( + network_info.get('inputs', []), self.inputs, self.input_shape, self.fit_to_input) + + def load_model(self, network_info, launcher, model_prefix=None, log=False): + self.network = launcher.create_network(network_info['model'], network_info['weights']) + self.load_network(self.network, launcher, model_prefix, network_info) + + def update_input_output_info(self, model_prefix, network_info): + def generate_name(prefix, with_prefix, layer_name): + return prefix + layer_name if with_prefix else layer_name.split(prefix)[-1] + + if model_prefix is None: + return + config_inputs = network_info.get('inputs', []) + network_with_prefix = False + if config_inputs: + config_with_prefix = config_inputs[0]['name'].startswith(model_prefix) + if config_with_prefix == network_with_prefix: + return + for c_input in config_inputs: + c_input['name'] = generate_name(model_prefix, network_with_prefix, c_input['name']) + network_info['inputs'] = config_inputs + config_outputs = network_info['outputs'] + for key, value in config_outputs.items(): + config_with_prefix = value.startswith(model_prefix) + if config_with_prefix != network_with_prefix: + config_outputs[key] = generate_name(model_prefix, network_with_prefix, value) + network_info['outputs'] = config_outputs class OVModelMixin(BaseOpenVINOModel): def _infer(self, input_blobs, batch_meta): @@ -743,6 +849,31 @@ def predict(self, input_blobs, batch_meta, output_callback=None): output_callback(out) return raw_outputs +class OpenCVProposalStage(OpenCVModelMixin, ProposalBaseStage): + def __init__( + self, model_info, model_specific_preprocessor, common_preprocessor, launcher, delayed_model_loading=False + ): + super().__init__(model_info, model_specific_preprocessor, common_preprocessor) + self.adapter = None + if not delayed_model_loading: + self.inputs = launcher.get_inputs_from_config(model_info) + model_xml, model_bin = self.prepare_model(self.default_model_name) + model_info.update({'model': model_xml, 'weights': model_bin}) + self.load_model(model_info, launcher, 'pnet_') + pnet_outs = model_info['outputs'] + pnet_adapter_config = launcher.config.get('adapter', {'type': 'mtcnn_p', **pnet_outs}) + # pnet_adapter_config.update({'regions_format': 'hw'}) + self.adapter = create_adapter(pnet_adapter_config) + + def input_shape(self, input_name): + return self.inputs[input_name] + + def predict(self, input_blobs, batch_meta, output_callback=None): + raw_outputs = self._infer(input_blobs, batch_meta) + if output_callback: + for out in raw_outputs: + output_callback(out) + return raw_outputs class OpenVINORefineStage(RefineBaseStage, OVModelMixin): def __init__( @@ -782,6 +913,30 @@ def predict(self, input_blobs, batch_meta, output_callback=None): output_callback(transform_for_callback(batch_size, raw_outputs)) return raw_outputs +class OpenCVRefineStage(OpenCVModelMixin, RefineBaseStage): + def __init__( + self, model_info, model_specific_preprocessor, common_preprocessor, launcher, delayed_model_loading=False + ): + super().__init__(model_info, model_specific_preprocessor, common_preprocessor) + if not delayed_model_loading: + self.inputs = launcher.get_inputs_from_config(model_info) + model_xml, model_bin = self.prepare_model(self.default_model_name) + model_info.update({'model': model_xml, 'weights': model_bin}) + self.load_model(model_info, launcher, 'pnet_') + pnet_outs = model_info['outputs'] + pnet_adapter_config = launcher.config.get('adapter', {'type': 'mtcnn_p', **pnet_outs}) + # pnet_adapter_config.update({'regions_format': 'hw'}) + self.adapter = create_adapter(pnet_adapter_config) + + def input_shape(self, input_name): + return self.inputs[input_name] + + def predict(self, input_blobs, batch_meta, output_callback=None): + raw_outputs = self._infer(input_blobs, batch_meta) + if output_callback: + batch_size = np.shape(next(iter(input_blobs[0].values())))[0] + output_callback(transform_for_callback(batch_size, raw_outputs)) + return raw_outputs class DLSDKOutputStage(DLSDKModelMixin, OutputBaseStage): def __init__( @@ -798,6 +953,28 @@ def predict(self, input_blobs, batch_meta, output_callback=None): raw_outputs = self._infer(input_blobs, batch_meta) return raw_outputs +class OpenCVOutputStage(OpenCVModelMixin, OutputBaseStage): + def __init__( + self, model_info, model_specific_preprocessor, common_preprocessor, launcher, delayed_model_loading=False + ): + super().__init__(model_info, model_specific_preprocessor, common_preprocessor) + if not delayed_model_loading: + self.inputs = launcher.get_inputs_from_config(model_info) + model_xml, model_bin = self.prepare_model(self.default_model_name) + model_info.update({'model': model_xml, 'weights': model_bin}) + self.load_model(model_info, launcher, 'pnet_') + pnet_outs = model_info['outputs'] + pnet_adapter_config = launcher.config.get('adapter', {'type': 'mtcnn_p', **pnet_outs}) + # pnet_adapter_config.update({'regions_format': 'hw'}) + self.adapter = create_adapter(pnet_adapter_config) + + def input_shape(self, input_name): + return self.inputs[input_name] + + def predict(self, input_blobs, batch_meta, output_callback=None): + raw_outputs = self._infer(input_blobs, batch_meta) + return raw_outputs + class OpenVINOOutputStage(OutputBaseStage, OVModelMixin): def __init__( diff --git a/tools/accuracy_checker/openvino/tools/accuracy_checker/launcher/opencv_launcher.py b/tools/accuracy_checker/openvino/tools/accuracy_checker/launcher/opencv_launcher.py index 60d01f8e496..99d299735f5 100644 --- a/tools/accuracy_checker/openvino/tools/accuracy_checker/launcher/opencv_launcher.py +++ b/tools/accuracy_checker/openvino/tools/accuracy_checker/launcher/opencv_launcher.py @@ -177,4 +177,5 @@ def release(self): """ Releases launcher. """ - del self.network + if 'network' in self.__dict__: + del self.network