Skip to content

Commit a83c12c

Browse files
vpisarevvpisarev
committed
Merge pull request opencv#10154 from alalek:ocl_cleanup_obsolete_cache
2 parents 7f554f3 + b6abf0d commit a83c12c

File tree

5 files changed

+230
-28
lines changed

5 files changed

+230
-28
lines changed

‎modules/core/include/opencv2/core/utils/filesystem.hpp

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,42 @@ namespace cv { namespace utils { namespace fs {
1111
CV_EXPORTS bool exists(const cv::String& path);
1212
CV_EXPORTS bool isDirectory(const cv::String& path);
1313

14+
CV_EXPORTS void remove_all(const cv::String& path);
15+
1416

1517
CV_EXPORTS cv::String getcwd();
1618

19+
/** Join path components */
20+
CV_EXPORTS cv::String join(const cv::String& base, const cv::String& path);
21+
22+
/**
23+
* Generate a list of all files that match the globbing pattern.
24+
*
25+
* Result entries are prefixed by base directory path.
26+
*
27+
* @param directory base directory
28+
* @param pattern filter pattern (based on '*'/'?' symbols). Use empty string to disable filtering and return all results
29+
* @param[out] result result of globing.
30+
* @param recursive scan nested directories too
31+
* @param includeDirectories include directories into results list
32+
*/
33+
CV_EXPORTS void glob(const cv::String& directory, const cv::String& pattern,
34+
CV_OUT std::vector<cv::String>& result,
35+
bool recursive = false, bool includeDirectories = false);
36+
37+
/**
38+
* Generate a list of all files that match the globbing pattern.
39+
*
40+
* @param directory base directory
41+
* @param pattern filter pattern (based on '*'/'?' symbols). Use empty string to disable filtering and return all results
42+
* @param[out] result globbing result with relative paths from base directory
43+
* @param recursive scan nested directories too
44+
* @param includeDirectories include directories into results list
45+
*/
46+
CV_EXPORTS void glob_relative(const cv::String& directory, const cv::String& pattern,
47+
CV_OUT std::vector<cv::String>& result,
48+
bool recursive = false, bool includeDirectories = false);
49+
1750

1851
CV_EXPORTS bool createDirectory(const cv::String& path);
1952
CV_EXPORTS bool createDirectories(const cv::String& path);

‎modules/core/include/opencv2/core/utils/logger.hpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -58,9 +58,9 @@ enum LogLevel {
5858
#endif
5959

6060

61-
#define CV_LOG_FATAL(tag, ...) for(;;) { std::stringstream ss; ss << "[FATAL:" << cv::utils::getThreadID() << "] " << __VA_ARGS__ << std::endl; std::cerr << ss.str(); break; }
62-
#define CV_LOG_ERROR(tag, ...) for(;;) { std::stringstream ss; ss << "[ERROR:" << cv::utils::getThreadID() << "] " << __VA_ARGS__ << std::endl; std::cerr << ss.str(); break; }
63-
#define CV_LOG_WARNING(tag, ...) for(;;) { std::stringstream ss; ss << "[ WARN:" << cv::utils::getThreadID() << "] " << __VA_ARGS__ << std::endl; std::cout << ss.str(); break; }
61+
#define CV_LOG_FATAL(tag, ...) for(;;) { std::stringstream ss; ss << "[FATAL:" << cv::utils::getThreadID() << "] " << __VA_ARGS__ << std::endl; std::cerr << ss.str() << std::flush; break; }
62+
#define CV_LOG_ERROR(tag, ...) for(;;) { std::stringstream ss; ss << "[ERROR:" << cv::utils::getThreadID() << "] " << __VA_ARGS__ << std::endl; std::cerr << ss.str() << std::flush; break; }
63+
#define CV_LOG_WARNING(tag, ...) for(;;) { std::stringstream ss; ss << "[ WARN:" << cv::utils::getThreadID() << "] " << __VA_ARGS__ << std::endl; std::cout << ss.str() << std::flush; break; }
6464
#if CV_LOG_STRIP_LEVEL <= CV_LOG_LEVEL_INFO
6565
#define CV_LOG_INFO(tag, ...)
6666
#else

‎modules/core/src/glob.cpp

Lines changed: 32 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,6 @@
4747
#if defined _WIN32 || defined WINCE
4848
# include <windows.h>
4949
const char dir_separators[] = "/\\";
50-
const char native_separator = '\\';
5150

5251
namespace
5352
{
@@ -136,7 +135,6 @@ namespace
136135
# include <dirent.h>
137136
# include <sys/stat.h>
138137
const char dir_separators[] = "/";
139-
const char native_separator = '/';
140138
#endif
141139

142140
static bool isDir(const cv::String& path, DIR* dir)
@@ -225,34 +223,36 @@ static bool wildcmp(const char *string, const char *wild)
225223
return *wild == 0;
226224
}
227225

228-
static void glob_rec(const cv::String& directory, const cv::String& wildchart, std::vector<cv::String>& result, bool recursive)
226+
static void glob_rec(const cv::String& directory, const cv::String& wildchart, std::vector<cv::String>& result,
227+
bool recursive, bool includeDirectories, const cv::String& pathPrefix)
229228
{
230229
DIR *dir;
231-
struct dirent *ent;
232230

233231
if ((dir = opendir (directory.c_str())) != 0)
234232
{
235233
/* find all the files and directories within directory */
236234
try
237235
{
236+
struct dirent *ent;
238237
while ((ent = readdir (dir)) != 0)
239238
{
240239
const char* name = ent->d_name;
241240
if((name[0] == 0) || (name[0] == '.' && name[1] == 0) || (name[0] == '.' && name[1] == '.' && name[2] == 0))
242241
continue;
243242

244-
cv::String path = directory + native_separator + name;
243+
cv::String path = cv::utils::fs::join(directory, name);
244+
cv::String entry = cv::utils::fs::join(pathPrefix, name);
245245

246246
if (isDir(path, dir))
247247
{
248248
if (recursive)
249-
glob_rec(path, wildchart, result, recursive);
250-
}
251-
else
252-
{
253-
if (wildchart.empty() || wildcmp(name, wildchart.c_str()))
254-
result.push_back(path);
249+
glob_rec(path, wildchart, result, recursive, includeDirectories, entry);
250+
if (!includeDirectories)
251+
continue;
255252
}
253+
254+
if (wildchart.empty() || wildcmp(name, wildchart.c_str()))
255+
result.push_back(entry);
256256
}
257257
}
258258
catch (...)
@@ -262,7 +262,10 @@ static void glob_rec(const cv::String& directory, const cv::String& wildchart, s
262262
}
263263
closedir(dir);
264264
}
265-
else CV_Error(CV_StsObjectNotFound, cv::format("could not open directory: %s", directory.c_str()));
265+
else
266+
{
267+
CV_Error_(CV_StsObjectNotFound, ("could not open directory: %s", directory.c_str()));
268+
}
266269
}
267270

268271
void cv::glob(String pattern, std::vector<String>& result, bool recursive)
@@ -298,6 +301,22 @@ void cv::glob(String pattern, std::vector<String>& result, bool recursive)
298301
}
299302
}
300303

301-
glob_rec(path, wildchart, result, recursive);
304+
glob_rec(path, wildchart, result, recursive, false, path);
305+
std::sort(result.begin(), result.end());
306+
}
307+
308+
void cv::utils::fs::glob(const cv::String& directory, const cv::String& pattern,
309+
std::vector<cv::String>& result,
310+
bool recursive, bool includeDirectories)
311+
{
312+
glob_rec(directory, pattern, result, recursive, includeDirectories, directory);
313+
std::sort(result.begin(), result.end());
314+
}
315+
316+
void cv::utils::fs::glob_relative(const cv::String& directory, const cv::String& pattern,
317+
std::vector<cv::String>& result,
318+
bool recursive, bool includeDirectories)
319+
{
320+
glob_rec(directory, pattern, result, recursive, includeDirectories, cv::String());
302321
std::sort(result.begin(), result.end());
303322
}

‎modules/core/src/ocl.cpp

Lines changed: 86 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,7 @@ void traceOpenCLCheck(cl_int status, const char* message)
180180
static const bool CV_OPENCL_CACHE_ENABLE = utils::getConfigurationParameterBool("OPENCV_OPENCL_CACHE_ENABLE", true);
181181
static const bool CV_OPENCL_CACHE_WRITE = utils::getConfigurationParameterBool("OPENCV_OPENCL_CACHE_WRITE", true);
182182
static const bool CV_OPENCL_CACHE_LOCK_ENABLE = utils::getConfigurationParameterBool("OPENCV_OPENCL_CACHE_LOCK_ENABLE", true);
183+
static const bool CV_OPENCL_CACHE_CLEANUP = utils::getConfigurationParameterBool("OPENCV_OPENCL_CACHE_CLEANUP", true);
183184

184185
#if CV_OPENCL_VALIDATE_BINARY_PROGRAMS
185186
static const bool CV_OPENCL_VALIDATE_BINARY_PROGRAMS_VALUE = utils::getConfigurationParameterBool("OPENCV_OPENCL_VALIDATE_BINARY_PROGRAMS", false);
@@ -254,6 +255,7 @@ struct OpenCLBinaryCacheConfigurator
254255

255256
typedef std::map<std::string, std::string> ContextCacheType;
256257
ContextCacheType prepared_contexts_;
258+
Mutex mutex_prepared_contexts_;
257259

258260
OpenCLBinaryCacheConfigurator()
259261
{
@@ -355,14 +357,17 @@ struct OpenCLBinaryCacheConfigurator
355357
cache_lock_.release();
356358
}
357359

358-
std::string prepareCacheDirectoryForContext(const std::string& ctx_prefix)
360+
std::string prepareCacheDirectoryForContext(const std::string& ctx_prefix,
361+
const std::string& cleanup_prefix)
359362
{
360363
if (cache_path_.empty())
361364
return std::string();
362365

363-
ContextCacheType::iterator i = prepared_contexts_.find(ctx_prefix);
364-
if (i != prepared_contexts_.end())
365-
return i->second;
366+
AutoLock lock(mutex_prepared_contexts_);
367+
368+
ContextCacheType::iterator found_it = prepared_contexts_.find(ctx_prefix);
369+
if (found_it != prepared_contexts_.end())
370+
return found_it->second;
366371

367372
CV_LOG_INFO(NULL, "Preparing OpenCL cache configuration for context: " << ctx_prefix);
368373

@@ -390,8 +395,59 @@ struct OpenCLBinaryCacheConfigurator
390395
target_directory = result ? target_directory : std::string();
391396
prepared_contexts_.insert(std::pair<std::string, std::string>(ctx_prefix, target_directory));
392397

393-
CV_LOG_VERBOSE(NULL, 1, " Result: " << (target_directory.empty() ? std::string("Failed") : target_directory));
398+
if (result && CV_OPENCL_CACHE_CLEANUP && CV_OPENCL_CACHE_WRITE && !cleanup_prefix.empty())
399+
{
400+
try
401+
{
402+
std::vector<String> entries;
403+
utils::fs::glob_relative(cache_path_, cleanup_prefix + "*", entries, false, true);
404+
std::vector<String> remove_entries;
405+
for (size_t i = 0; i < entries.size(); i++)
406+
{
407+
const String& name = entries[i];
408+
if (0 == name.find(cleanup_prefix))
409+
{
410+
if (0 == name.find(ctx_prefix))
411+
continue; // skip current
412+
remove_entries.push_back(name);
413+
}
414+
}
415+
if (!remove_entries.empty())
416+
{
417+
CV_LOG_WARNING(NULL, (remove_entries.size() == 1
418+
? "Detected OpenCL cache directory for other version of OpenCL device."
419+
: "Detected OpenCL cache directories for other versions of OpenCL device.")
420+
<< " We assume that these directories are obsolete after OpenCL runtime/drivers upgrade.");
421+
CV_LOG_WARNING(NULL, "Trying to remove these directories...");
422+
for (size_t i = 0; i < remove_entries.size(); i++)
423+
{
424+
CV_LOG_WARNING(NULL, "- " << remove_entries[i]);
425+
}
426+
CV_LOG_WARNING(NULL,"Note: You can disable this behavior via this option: CV_OPENCL_CACHE_CLEANUP=0");
427+
428+
for (size_t i = 0; i < remove_entries.size(); i++)
429+
{
430+
const String& name = remove_entries[i];
431+
cv::String path = utils::fs::join(cache_path_, name);
432+
try
433+
{
434+
utils::fs::remove_all(path);
435+
CV_LOG_WARNING(NULL, "Removed: " << path);
436+
}
437+
catch (const cv::Exception& e)
438+
{
439+
CV_LOG_ERROR(NULL, "Exception during removal of obsolete OpenCL cache directory: " << path << std::endl << e.what());
440+
}
441+
}
442+
}
443+
}
444+
catch (...)
445+
{
446+
CV_LOG_WARNING(NULL, "Can't check for obsolete OpenCL cache directories");
447+
}
448+
}
394449

450+
CV_LOG_VERBOSE(NULL, 1, " Result: " << (target_directory.empty() ? std::string("Failed") : target_directory));
395451
return target_directory;
396452
}
397453

@@ -1969,7 +2025,7 @@ struct Context::Impl
19692025
}
19702026
}
19712027

1972-
std::string getPrefixString()
2028+
std::string& getPrefixString()
19732029
{
19742030
if (prefix.empty())
19752031
{
@@ -1988,12 +2044,32 @@ struct Context::Impl
19882044
return prefix;
19892045
}
19902046

2047+
std::string& getPrefixBase()
2048+
{
2049+
if (prefix_base.empty())
2050+
{
2051+
const Device& d = devices[0];
2052+
prefix_base = d.vendorName() + "--" + d.name() + "--";
2053+
// sanitize chars
2054+
for (size_t i = 0; i < prefix_base.size(); i++)
2055+
{
2056+
char c = prefix_base[i];
2057+
if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || c == '-'))
2058+
{
2059+
prefix_base[i] = '_';
2060+
}
2061+
}
2062+
}
2063+
return prefix_base;
2064+
}
2065+
19912066
IMPLEMENT_REFCOUNTABLE();
19922067

19932068
cl_context handle;
19942069
std::vector<Device> devices;
19952070

19962071
std::string prefix;
2072+
std::string prefix_base;
19972073

19982074
cv::Mutex program_cache_mutex;
19992075
typedef std::map<std::string, Program> phash_t;
@@ -3233,7 +3309,10 @@ struct Program::Impl
32333309
{
32343310
#if OPENCV_HAVE_FILESYSTEM_SUPPORT
32353311
OpenCLBinaryCacheConfigurator& config = OpenCLBinaryCacheConfigurator::getSingletonInstance();
3236-
const std::string base_dir = config.prepareCacheDirectoryForContext(ctx.getImpl()->getPrefixString());
3312+
const std::string base_dir = config.prepareCacheDirectoryForContext(
3313+
ctx.getImpl()->getPrefixString(),
3314+
ctx.getImpl()->getPrefixBase()
3315+
);
32373316
const std::string fname = base_dir.empty() ? std::string() :
32383317
std::string(base_dir + src.getImpl()->module_.c_str() + "--" + src.getImpl()->name_ + "_" + src.getImpl()->codeHash_ + ".bin");
32393318
const cv::Ptr<utils::fs::FileLock> fileLock = config.cache_lock_; // can be empty

0 commit comments

Comments
 (0)