Skip to content

Commit

Permalink
AUTO/MULTI supports ov::auto_batch_timeout (openvinotoolkit#12023)
Browse files Browse the repository at this point in the history 
* add auto_batch_timeout for MULTI and AUTO

* fix clang-format for ie_core.cpp

* fix coredump

* simplify insert key to deviceConfig logic and parseDeviceNameIntoConfig() check "AUTO" and "AUTO:" only

* check config auto_batch_timeout

* add CleanUpInIECore()

* fix clang-format for ie_core.cpp
  • Loading branch information
@wgzintel
wgzintel authored Jul 7, 2022
1 parent d5e8d1d commit cd6c7da
Show file tree
Hide file tree
Showing 2 changed files with 48 additions and 6 deletions.
15 changes: 14 additions & 1 deletion src/inference/src/ie_core.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ Parsed<T> parseDeviceNameIntoConfig(const std::string& deviceName, const std::ma
} else if (deviceName_.find("MULTI:") == 0) {
deviceName_ = "MULTI";
config_[ie::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES] = deviceName.substr(6);
} else if (deviceName.find("AUTO") == 0) {
} else if (deviceName == "AUTO" || deviceName.find("AUTO:") == 0) {
deviceName_ = "AUTO";
if (deviceName.find("AUTO:") == 0) {
config_[ie::MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES] =
Expand Down Expand Up @@ -615,6 +615,7 @@ class CoreImpl : public ie::ICore, public std::enable_shared_from_this<ie::ICore
std::map<std::string, std::string>& config_with_batch = parsed._config;
// if auto-batching is applicable, the below function will patch the device name and config accordingly:
ApplyAutoBatching(network, deviceName, config_with_batch);
CleanUpProperties(deviceName, config_with_batch);
parsed = parseDeviceNameIntoConfig(deviceName, config_with_batch);

auto plugin = GetCPPPluginByName(parsed._deviceName);
Expand Down Expand Up @@ -701,6 +702,17 @@ class CoreImpl : public ie::ICore, public std::enable_shared_from_this<ie::ICore
}
}

void CleanUpProperties(std::string& deviceName, std::map<std::string, std::string>& config) {
// auto-batching is not applicable, if there is auto_batch_timeout, delete it
if (deviceName.find("BATCH") == std::string::npos) {
const auto& batch_timeout_mode = config.find(ov::auto_batch_timeout.name());
if (batch_timeout_mode != config.end()) {
if (deviceName.find("AUTO") == std::string::npos && deviceName.find("MULTI") == std::string::npos)
config.erase(batch_timeout_mode);
}
}
}

ie::SoExecutableNetworkInternal LoadNetwork(const ie::CNNNetwork& network,
const std::string& deviceNameOrig,
const std::map<std::string, std::string>& config) override {
Expand All @@ -709,6 +721,7 @@ class CoreImpl : public ie::ICore, public std::enable_shared_from_this<ie::ICore
std::map<std::string, std::string> config_with_batch = config;
// if auto-batching is applicable, the below function will patch the device name and config accordingly:
ApplyAutoBatching(network, deviceName, config_with_batch);
CleanUpProperties(deviceName, config_with_batch);

bool forceDisableCache = config_with_batch.count(CONFIG_KEY_INTERNAL(FORCE_DISABLE_CACHE)) > 0;
auto parsed = parseDeviceNameIntoConfig(deviceName, config_with_batch);
Expand Down
39 changes: 34 additions & 5 deletions src/plugins/auto/plugin.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ namespace {
res.push_back(ov::hint::allow_auto_batching.name());
res.push_back(ov::log::level.name());
res.push_back(ov::intel_auto::device_bind_buffer.name());
res.push_back(ov::auto_batch_timeout.name());
return res;
}();
} // namespace
Expand Down Expand Up @@ -269,6 +270,7 @@ InferenceEngine::Parameter MultiDeviceInferencePlugin::GetMetric(const std::stri
RW_property(ov::device::priorities.name()),
RW_property(ov::enable_profiling.name()),
RW_property(ov::hint::allow_auto_batching.name()),
RW_property(ov::auto_batch_timeout.name()),
RW_property(ov::hint::performance_mode.name()),
RW_property(ov::hint::num_requests.name())
};
Expand Down Expand Up @@ -343,7 +345,25 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons
std::vector<DeviceInformation> metaDevices;
bool workModeAuto = GetName() == "AUTO";
auto priorities = fullConfig.find(MultiDeviceConfigParams::KEY_MULTI_DEVICE_PRIORITIES);
// If the user sets the property, insert the property into the deviceConfig
auto insertPropToConfig = [&](std::string property,
std::string& deviceName,
std::map<std::string, std::string>& deviceConfig) {
auto tmpiter =
std::find_if(fullConfig.begin(), fullConfig.end(), [&](const std::pair<std::string, std::string>& config) {
return (config.first == property);
});
if (tmpiter != fullConfig.end()) {
deviceConfig.insert({tmpiter->first, tmpiter->second});
LOG_INFO_TAG("device:%s, config:%s=%s",
deviceName.c_str(),
tmpiter->first.c_str(),
tmpiter->second.c_str());
}
};

// if workMode is AUTO
// only AUTO uses CheckConfig() to check fullConfig's parameters, MULTI does not
if (workModeAuto) {
// check the configure and check if need to set PerfCounters configure to device
// and set filter configure
Expand Down Expand Up @@ -396,11 +416,8 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons
config.first.c_str(), config.second.c_str());
}
}
auto tmpiter = std::find_if(fullConfig.begin(), fullConfig.end(), [](const std::pair<std::string, std::string>& config) {
return (config.first == CONFIG_KEY(ALLOW_AUTO_BATCHING));
});
if (tmpiter != fullConfig.end())
deviceConfig.insert({tmpiter->first, tmpiter->second});
insertPropToConfig(CONFIG_KEY(ALLOW_AUTO_BATCHING), iter->deviceName, deviceConfig);
insertPropToConfig(CONFIG_KEY(AUTO_BATCH_TIMEOUT), iter->deviceName, deviceConfig);
iter->config = deviceConfig;
strDevices += iter->deviceName;
strDevices += ((iter + 1) == supportDevices.end()) ? "" : ",";
Expand Down Expand Up @@ -450,6 +467,7 @@ IExecutableNetworkInternal::Ptr MultiDeviceInferencePlugin::LoadNetworkImpl(cons
multiSContext->_batchingDisabled = true;
p.config.insert({tmpiter->first, tmpiter->second});
}
insertPropToConfig(CONFIG_KEY(AUTO_BATCH_TIMEOUT), p.deviceName, p.config);
const auto& deviceName = p.deviceName;
const auto& deviceConfig = p.config;
SoExecutableNetworkInternal exec_net;
Expand Down Expand Up @@ -846,6 +864,17 @@ void MultiDeviceInferencePlugin::CheckConfig(const std::map<std::string, std::st
context->_batchingDisabled = true;
continue;
}
} else if (kvp.first == ov::auto_batch_timeout) {
try {
auto batch_timeout = std::stoi(kvp.second);
if (batch_timeout < 0) {
IE_THROW() << "Unsupported config value: " << kvp.second
<< " for key: " << kvp.first;
}
} catch (...) {
IE_THROW() << "Unsupported config value: " << kvp.second
<< " for key: " << kvp.first;
}
} else if (kvp.first == ov::intel_auto::device_bind_buffer.name()) {
if (kvp.second == PluginConfigParams::YES ||
kvp.second == PluginConfigParams::NO) {
Expand Down

0 comments on commit cd6c7da

Please sign in to comment.