Skip to content

Commit

Permalink
Cleanup for log lvl and terms used
Browse files Browse the repository at this point in the history
  • Loading branch information
ethan-eb committed Feb 1, 2013
1 parent ccfc6d1 commit 6a01fb3
Showing 1 changed file with 16 additions and 15 deletions.
31 changes: 16 additions & 15 deletions tablechop
Original file line number Diff line number Diff line change
Expand Up @@ -43,59 +43,60 @@ def clean_backups(args, log):
key_list = sorted(key_list, key=lambda k: dtparser.parse(k.last_modified))
key_list.reverse() # most recent first

log.debug("%s keys total", len(key_list))
log.info("%s keys total", len(key_list))

json_files = [x for x in key_list if x.name.endswith('-listdir.json')]
json_keys = [x for x in key_list if x.name.endswith('-listdir.json')]
to_delete = set([x.name for x in key_list]) # we'll remove from this list

log.debug("%s json listdir files", len(json_files))
log.debug("%s json listdir keys", len(json_keys))

for jfile in json_files:
log.debug("file dated : %s (%s)", jfile.last_modified,
jfile.name.split('/')[-1])
if days_ago(jfile.last_modified) > args.age:
for jkey in json_keys:
log.debug("key dated : %s (%s)", jkey.last_modified,
jkey.name.split('/')[-1])
if days_ago(jkey.last_modified) > args.age:
# We've gone back past our cutoff
log.info("reached cutoff at timestamp %s", jfile.last_modified)
log.info("reached cutoff at timestamp %s", jkey.last_modified)
break
ky = bucket.get_key(jfile)
ky = bucket.get_key(jkey)
jdict = json.loads(ky.get_contents_as_string())
if len(jdict.values()) != 1:
raise SystemError('-listdir.json file should have '
'a single key/value pair!')
# formatting here to match the full 'key' version of filename
keepers = set(["%s/%s" % (args.path, x) for x in jdict.values()[0]])
keepers.add(jfile.name) # keep this json file itself
keepers.add(jkey.name) # keep this json file itself
to_delete = to_delete.difference(keepers)

if args.debug:
log.info("%s non-keeper files to delete", len(to_delete))
log.debug("%s non-keeper keys to delete", len(to_delete))
ddates = list(set([x.last_modified[:10] for x in key_list \
if x.name in to_delete]))
ddates.sort()
log.debug("deletion dates : %s", ddates)
log.debug("Test mode, nothing deleted")
return

log.info("Deleting %s keys" % len(to_delete))

try:
bucket.delete_keys(to_delete) # !!
except Exception as e:
log.error('S3 delete ERR, will try again later [%s]', e)

log.info("Keys deleted")


def main(log):
parser = argparse.ArgumentParser(
description='Clean SSTables from S3. Scroll backwards through '
'-listdir.json files in chronological order collecting a "keeper" '
'list until it reaches it\'s age cutoff. Deletes all files not in that '
'-listdir.json keys in chronological order collecting a "keeper" '
'list until it reaches it\'s age cutoff. Deletes all keys not in that '
'list')
parser.add_argument(
'-d',
'--debug',
dest='debug',
action='store_true',
help='Run in debug mode, will not delete files. Implies -v')
help='Run in debug mode, will not delete keys. Implies -v')
parser.add_argument(
'-k',
'--key',
Expand Down

0 comments on commit 6a01fb3

Please sign in to comment.