diff --git a/src/inference/src/ie_core.cpp b/src/inference/src/ie_core.cpp index 21bacc2fa8cee7..28cb707f5c7bd4 100644 --- a/src/inference/src/ie_core.cpp +++ b/src/inference/src/ie_core.cpp @@ -96,7 +96,7 @@ Parsed parseDeviceNameIntoConfig(const std::string& deviceName, const std::ma } else if (deviceName_.find("MULTI:") == 0) { deviceName_ = "MULTI"; config_[ie::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES] = deviceName.substr(6); - } else if (deviceName.find("AUTO") == 0) { + } else if (deviceName == "AUTO" || deviceName.find("AUTO:") == 0) { deviceName_ = "AUTO"; if (deviceName.find("AUTO:") == 0) { config_[ie::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES] = @@ -615,6 +615,7 @@ class CoreImpl : public ie::ICore, public std::enable_shared_from_this& config_with_batch = parsed._config; // if auto-batching is applicable, the below function will patch the device name and config accordingly: ApplyAutoBatching(network, deviceName, config_with_batch); + CleanUpProperties(deviceName, config_with_batch); parsed = parseDeviceNameIntoConfig(deviceName, config_with_batch); auto plugin = GetCPPPluginByName(parsed._deviceName); @@ -701,6 +702,17 @@ class CoreImpl : public ie::ICore, public std::enable_shared_from_this& config) { + // auto-batching is not applicable, if there is auto_batch_timeout, delete it + if (deviceName.find("BATCH") == std::string::npos) { + const auto& batch_timeout_mode = config.find(ov::auto_batch_timeout.name()); + if (batch_timeout_mode != config.end()) { + if (deviceName.find("AUTO") == std::string::npos && deviceName.find("MULTI") == std::string::npos) + config.erase(batch_timeout_mode); + } + } + } + ie::SoExecutableNetworkInternal LoadNetwork(const ie::CNNNetwork& network, const std::string& deviceNameOrig, const std::map& config) override { @@ -709,6 +721,7 @@ class CoreImpl : public ie::ICore, public std::enable_shared_from_this config_with_batch = config; // if auto-batching is applicable, the below function will patch the device name and config accordingly: ApplyAutoBatching(network, deviceName, config_with_batch); + CleanUpProperties(deviceName, config_with_batch); bool forceDisableCache = config_with_batch.count(CONFIG_KEY_INTERNAL(FORCE_DISABLE_CACHE)) > 0; auto parsed = parseDeviceNameIntoConfig(deviceName, config_with_batch); diff --git a/src/plugins/auto/plugin.cpp b/src/plugins/auto/plugin.cpp index 651c3d14d9ebc7..9ff783c88a2bb8 100644 --- a/src/plugins/auto/plugin.cpp +++ b/src/plugins/auto/plugin.cpp @@ -69,6 +69,7 @@ namespace { res.push_back(ov::hint::allow_auto_batching.name()); res.push_back(ov::log::level.name()); res.push_back(ov::intel_auto::device_bind_buffer.name()); + res.push_back(ov::auto_batch_timeout.name()); return res; }(); } // namespace @@ -269,6 +270,7 @@ InferenceEngine::Parameter MultiDeviceInferencePlugin::GetMetric(const std::stri RW_property(ov::device::priorities.name()), RW_property(ov::enable_profiling.name()), RW_property(ov::hint::allow_auto_batching.name()), + RW_property(ov::auto_batch_timeout.name()), RW_property(ov::hint::performance_mode.name()), RW_property(ov::hint::num_requests.name()) }; @@ -343,7 +345,25 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons std::vector metaDevices; bool workModeAuto = GetName() == "AUTO"; auto priorities = fullConfig.find(MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES); + // If the user sets the property, insert the property into the deviceConfig + auto insertPropToConfig = [&](std::string property, + std::string& deviceName, + std::map& deviceConfig) { + auto tmpiter = + std::find_if(fullConfig.begin(), fullConfig.end(), [&](const std::pair& config) { + return (config.first == property); + }); + if (tmpiter != fullConfig.end()) { + deviceConfig.insert({tmpiter->first, tmpiter->second}); + LOG_INFO_TAG("device:%s, config:%s=%s", + deviceName.c_str(), + tmpiter->first.c_str(), + tmpiter->second.c_str()); + } + }; + // if workMode is AUTO + // only AUTO uses CheckConfig() to check fullConfig's parameters, MULTI does not if (workModeAuto) { // check the configure and check if need to set PerfCounters configure to device // and set filter configure @@ -396,11 +416,8 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons config.first.c_str(), config.second.c_str()); } } - auto tmpiter = std::find_if(fullConfig.begin(), fullConfig.end(), [](const std::pair& config) { - return (config.first == CONFIG_KEY(ALLOW_AUTO_BATCHING)); - }); - if (tmpiter != fullConfig.end()) - deviceConfig.insert({tmpiter->first, tmpiter->second}); + insertPropToConfig(CONFIG_KEY(ALLOW_AUTO_BATCHING), iter->deviceName, deviceConfig); + insertPropToConfig(CONFIG_KEY(AUTO_BATCH_TIMEOUT), iter->deviceName, deviceConfig); iter->config = deviceConfig; strDevices += iter->deviceName; strDevices += ((iter + 1) == supportDevices.end()) ? "" : ","; @@ -450,6 +467,7 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons multiSContext->_batchingDisabled = true; p.config.insert({tmpiter->first, tmpiter->second}); } + insertPropToConfig(CONFIG_KEY(AUTO_BATCH_TIMEOUT), p.deviceName, p.config); const auto& deviceName = p.deviceName; const auto& deviceConfig = p.config; SoExecutableNetworkInternal exec_net; @@ -846,6 +864,17 @@ void MultiDeviceInferencePlugin::CheckConfig(const std::map_batchingDisabled = true; continue; } + } else if (kvp.first == ov::auto_batch_timeout) { + try { + auto batch_timeout = std::stoi(kvp.second); + if (batch_timeout < 0) { + IE_THROW() << "Unsupported config value: " << kvp.second + << " for key: " << kvp.first; + } + } catch (...) { + IE_THROW() << "Unsupported config value: " << kvp.second + << " for key: " << kvp.first; + } } else if (kvp.first == ov::intel_auto::device_bind_buffer.name()) { if (kvp.second == PluginConfigParams::YES || kvp.second == PluginConfigParams::NO) {