Skip to content

Commit

Permalink
Merge pull request mantidproject#34704 from mantidproject/debug-clean…
Browse files Browse the repository at this point in the history 
…filecache

Add debugging print statements for CleanFileCache
  • Loading branch information
@martyngigg
martyngigg authored Nov 10, 2022
2 parents b71e2a4 + f280d1c commit 850bfc9
Show file tree
Hide file tree
Showing 2 changed files with 62 additions and 57 deletions.
32 changes: 18 additions & 14 deletions Framework/PythonInterface/plugins/algorithms/CleanFileCache.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,12 @@
import os


# See ticket #14716
def _print_while_testing(msg: str) -> None:
"""If we are testing then print out extra information"""
# This is currently keyed to the Jenkins names that are known
if "JOB_NAME" in os.environ:
print(msg, flush=True)


class CleanFileCache(PythonAlgorithm):
""" Remove cache files from the cache directory
Expand All @@ -22,7 +27,7 @@ def category(self):
return "Workflow\\DataHandling"

def seeAlso(self):
return [ "ClearCache" ]
return ["ClearCache"]

def name(self):
"""
Expand All @@ -49,10 +54,8 @@ def PyInit(self):
"the directory in which the cache file will be created. If nothing is given, default location for cache files will be used",
Direction.Input)

self.declareProperty(
"AgeInDays", 14,
"If any file is more than this many days old, it will be deleted. 0 means remove everything",
Direction.Input)
self.declareProperty("AgeInDays", 14, "If any file is more than this many days old, it will be deleted. 0 means remove everything",
Direction.Input)
return

def PyExec(self):
Expand All @@ -61,10 +64,7 @@ def PyExec(self):
# Inputs
cache_dir = self.getPropertyValue("CacheDir")
if not cache_dir:
cache_dir = os.path.join(
ConfigService.getUserPropertiesDir(),
"cache"
)
cache_dir = os.path.join(ConfigService.getUserPropertiesDir(), "cache")
age = int(self.getPropertyValue("AgeInDays"))
#
_run(cache_dir, age)
Expand All @@ -76,22 +76,26 @@ def _run(cache_dir, days):
import re
import time
from datetime import timedelta, date
rm_date = date.today() - timedelta(days = days)
rm_date = time.mktime(rm_date.timetuple()) + 24*60*60
rm_date = date.today() - timedelta(days=days)
rm_date = time.mktime(rm_date.timetuple()) + 24 * 60 * 60
for f in glob.glob(os.path.join(cache_dir, "*.nxs")):
# skip over non-files
if not os.path.isfile(f):
continue
# skip over new files
# print os.stat(f).st_mtime, rm_date
if os.stat(f).st_mtime > rm_date:
mtime = os.stat(f).st_mtime
_print_while_testing(f"{f}: mtime={mtime}, rm_date={rm_date}")
if mtime > rm_date:
_print_while_testing(f" skipping {f}")
continue
# check filename pattern
base = os.path.basename(f)
if re.match(".*_[0-9a-f]{40}.nxs", base):
_print_while_testing(f" removing {f}")
os.remove(f)
continue
if re.match("[0-9a-f]{40}.nxs", base):
_print_while_testing(f" removing {f}")
os.remove(f)
continue
continue
Expand Down
87 changes: 44 additions & 43 deletions Framework/PythonInterface/test/python/plugins/algorithms/CleanFileCacheTest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,18 +24,16 @@ def test1(self):
# and other files
cache_root = tempfile.mkdtemp()
_hash = lambda s: hashlib.sha1(s).hexdigest()
cache1, _ = CreateCacheFilename(
CacheDir = cache_root,
OtherProperties = ["A=1", "B=2"]
)
cache1, _ = CreateCacheFilename(CacheDir=cache_root, OtherProperties=["A=1", "B=2"])
cache2, _ = CreateCacheFilename(
CacheDir = cache_root,
OtherProperties = ["C=3"],
CacheDir=cache_root,
OtherProperties=["C=3"],
)
touch(cache1)
touch(cache2)
non_cache = [os.path.join(cache_root, f) for f in ["normal1.txt", "normal2.dat"]]
for p in non_cache: touch(p)
for p in non_cache:
touch(p)
# print glob.glob(os.path.join(cache_root, '*'))
# Execute
code = "CleanFileCache(CacheDir = %r, AgeInDays = 0)" % cache_root
Expand All @@ -54,33 +52,30 @@ def test1(self):
shutil.rmtree(cache_root)
return


def test2(self):
"""CleanFileCache: 'normal' files with 39 and 41-character filenames etc
"""
# create a temporary directory with fake cache files
# and other files
cache_root = tempfile.mkdtemp()
_hash = lambda s: hashlib.sha1(s).hexdigest()
cache1, _ = CreateCacheFilename(
CacheDir = cache_root,
OtherProperties = ["A=1"]
)
cache1, _ = CreateCacheFilename(CacheDir=cache_root, OtherProperties=["A=1"])
cache2, _ = CreateCacheFilename(
CacheDir = cache_root,
OtherProperties = ["B='silly'"],
CacheDir=cache_root,
OtherProperties=["B='silly'"],
)
touch(cache1)
touch(cache2)
non_cache = [
os.path.join(cache_root, f)
for f in [
'a'*39+".nxs", '0'*41+".nxs",
'alpha_' + 'b'*39 + ".nxs",
'beta_' + 'e'*41 + ".nxs",
os.path.join(cache_root, f) for f in [
'a' * 39 + ".nxs",
'0' * 41 + ".nxs",
'alpha_' + 'b' * 39 + ".nxs",
'beta_' + 'e' * 41 + ".nxs",
]
]
for p in non_cache: touch(p)
for p in non_cache:
touch(p)
# print glob.glob(os.path.join(cache_root, '*'))
# Execute
code = "CleanFileCache(CacheDir = %r, AgeInDays = 0)" % cache_root
Expand All @@ -99,7 +94,6 @@ def test2(self):
shutil.rmtree(cache_root)
return


def test3(self):
"""CleanFileCache: "age" parameter
"""
Expand All @@ -108,31 +102,36 @@ def test3(self):
# and other files
cache_root = tempfile.mkdtemp()
_hash = lambda s: hashlib.sha1(s).hexdigest()
cache1, _ = CreateCacheFilename(
CacheDir = cache_root,
OtherProperties = ["A=newer"]
)
cache1, _ = CreateCacheFilename(CacheDir=cache_root, OtherProperties=["A=newer"])
cache2, _ = CreateCacheFilename(
CacheDir = cache_root,
OtherProperties = ["B=rightonedge"],
CacheDir=cache_root,
OtherProperties=["B=rightonedge"],
)
cache3, _ = CreateCacheFilename(
CacheDir = cache_root,
OtherProperties = ["C=old"],
CacheDir=cache_root,
OtherProperties=["C=old"],
)
createFile(cache1, age-1)
createFile(cache2, age)
createFile(cache3, age+1)
print(f"Time before creating files: {time.time()}", flush=True)
print(f"cache1={cache1}", flush=True)
createFile(cache1, age - 1, display=True)
print(f"cache2={cache2}", flush=True)
createFile(cache2, age, display=True)
print(f"cache3={cache3}", flush=True)
createFile(cache3, age + 1, display=True)
non_cache = [
os.path.join(cache_root, f)
for f in [
'a'*39+".nxs", '0'*41+".nxs",
'alpha_' + 'b'*39 + ".nxs",
'beta_' + 'e'*41 + ".nxs",
os.path.join(cache_root, f) for f in [
'a' * 39 + ".nxs",
'0' * 41 + ".nxs",
'alpha_' + 'b' * 39 + ".nxs",
'beta_' + 'e' * 41 + ".nxs",
]
]
for p in non_cache: touch(p)
# print glob.glob(os.path.join(cache_root, '*'))
for p in non_cache:
touch(p)
print("Cache files:", flush=True)
print(glob.glob(os.path.join(cache_root, '*')), flush=True)
print(f"Time before running CleanFileCache algorithm: {time.time()}", flush=True)

# Execute
code = "CleanFileCache(CacheDir = %r, AgeInDays = %s)" % (cache_root, age)
code = "from mantid.simpleapi import CleanFileCache; %s" % code
Expand All @@ -144,25 +143,27 @@ def test3(self):
# Verify ....
files_remained = glob.glob(os.path.join(cache_root, '*'))
try:
self.assertEqual(set(files_remained), set(non_cache+[cache1]))
self.assertEqual(set(files_remained), set(non_cache + [cache1]))
finally:
# remove the temporary directory
shutil.rmtree(cache_root)
return


def createFile(f, daysbefore):
def createFile(f, daysbefore, display=False):
"create a file and set modify time at n=daysbefore days before today"
touch(f)
t = computeTime(daysbefore)
os.utime(f, (t,t))
if display:
print(f" Computed modification time: {t}")
os.utime(f, (t, t))
# print time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(os.stat(f).st_mtime))
return


def computeTime(daysbefore):
"compute time as float of the time at n=daysbefore days before today"
return time.time() - daysbefore * 24*60*60
return time.time() - daysbefore * 24 * 60 * 60


def touch(f):
Expand Down

0 comments on commit 850bfc9

Please sign in to comment.