Skip to content

Commit

Permalink
add only profile purge option
Browse files Browse the repository at this point in the history
  • Loading branch information
@gunlee01
gunlee01 committed May 6, 2017
1 parent dca754b commit 5f21867
Show file tree
Hide file tree
Showing 2 changed files with 88 additions and 25 deletions.
15 changes: 9 additions & 6 deletions scouter.server/src/scouter/server/Configure.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,12 +172,14 @@ public final static synchronized Configure getInstance() {
@Deprecated
@ConfigDesc("Deprecated. Use option mgr_purge_non_xlog_keep_days")
public boolean mgr_purge_only_xlog_enabled = false;
@ConfigDesc("Condition of disc usage for automatic deletion")
@ConfigDesc("Condition of disk usage for automatic deletion. if lack, delete profile data first exclude today data.")
public int mgr_purge_disk_usage_pct = 80;
@ConfigDesc("Retaining date for automatic deletion. delete profile data first.")
public int mgr_purge_keep_days = 10;
@ConfigDesc("Retaining date for automatic deletion.")
public int mgr_purge_xlog_without_profile_keep_days = 30;
@ConfigDesc("Retaining date for automatic deletion")
public int mgr_purge_keep_days = 0;
@ConfigDesc("Retaining date for automatic deletion")
public int mgr_purge_non_xlog_keep_days = 0;
public int mgr_purge_counter_keep_days = 70;
@ConfigDesc("Ignored log ID set")
public StringSet mgr_log_ignore_ids = new StringSet();

Expand Down Expand Up @@ -353,8 +355,9 @@ private void apply() {
this.mgr_purge_enabled = getBoolean("mgr_purge_enabled", true);
this.mgr_purge_only_xlog_enabled = getBoolean("mgr_purge_only_xlog_enabled", false);
this.mgr_purge_disk_usage_pct = getInt("mgr_purge_disk_usage_pct", 80);
this.mgr_purge_keep_days = getInt("mgr_purge_keep_days", 0);
this.mgr_purge_non_xlog_keep_days = getInt("mgr_purge_non_xlog_keep_days", mgr_purge_keep_days*5);
this.mgr_purge_keep_days = getInt("mgr_purge_keep_days", 10);
this.mgr_purge_xlog_without_profile_keep_days = getInt("mgr_purge_xlog_keep_days", mgr_purge_keep_days*3);
this.mgr_purge_counter_keep_days = getInt("mgr_purge_non_xlog_keep_days", mgr_purge_keep_days*7);

this.mgr_text_db_daily_service_enabled = getBoolean("mgr_text_db_daily_service_enabled", false);
this.mgr_text_db_daily_api_enabled = getBoolean("mgr_text_db_daily_api_enabled", false);
Expand Down
98 changes: 79 additions & 19 deletions scouter.server/src/scouter/server/core/AutoDeleteScheduler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,16 +18,21 @@
import java.util.Set;

public class AutoDeleteScheduler extends Thread {
enum Mode {
PROFILE, XLOG, ALL,;
}

private static AutoDeleteScheduler instance = null;
private static final long CHECK_INTERVAL = DateUtil.MILLIS_PER_MINUTE;
Configure conf = Configure.getInstance();
boolean brun;
int maxPercent;
int retainDays;
int retainNonXLogDays;
int retainXlogDays;
int retainCounterDays;
boolean delOnlyXLog;
String lastCheckDate;
Set<String> deletedProfileDays = new HashSet<String>();
Set<String> deletedXLogDays = new HashSet<String>();
Set<String> deletedDays = new HashSet<String>();

Expand All @@ -51,11 +56,14 @@ public void run() {
if (conf.mgr_purge_enabled != brun
|| conf.mgr_purge_disk_usage_pct != maxPercent
|| conf.mgr_purge_keep_days != retainDays
|| conf.mgr_purge_non_xlog_keep_days != retainNonXLogDays
|| conf.mgr_purge_xlog_without_profile_keep_days != retainXlogDays
|| conf.mgr_purge_counter_keep_days != retainCounterDays
|| conf.mgr_purge_only_xlog_enabled != delOnlyXLog) {
applyConf();
lastCheckDate = null;
deletedProfileDays.clear();
deletedXLogDays.clear();
deletedDays.clear();
}
}
});
Expand All @@ -65,7 +73,8 @@ private void applyConf() {
brun = conf.mgr_purge_enabled;
maxPercent = conf.mgr_purge_disk_usage_pct;
retainDays = conf.mgr_purge_keep_days;
retainNonXLogDays = conf.mgr_purge_non_xlog_keep_days;
retainXlogDays = conf.mgr_purge_xlog_without_profile_keep_days;
retainCounterDays = conf.mgr_purge_counter_keep_days;
delOnlyXLog = conf.mgr_purge_only_xlog_enabled;
}

Expand All @@ -78,24 +87,64 @@ public void run() {
if (maxPercent > 0) {
long totalSpace = dbDir.getTotalSpace();
long usuableSpace = dbDir.getUsableSpace();
double percent = (usuableSpace * 100.0d) / totalSpace;
while ((100 - percent) > maxPercent) {
double freePercent = (usuableSpace * 100.0d) / totalSpace;
while ((100 - freePercent) > maxPercent) {
String yyyymmdd = getLongAgoDate(deletedProfileDays);
if (yyyymmdd == null || today.equals(yyyymmdd)) {
break;
}
deleteData(yyyymmdd, Mode.PROFILE);
usuableSpace = dbDir.getUsableSpace();
freePercent = (usuableSpace * 100.0d) / totalSpace;
deletedProfileDays.add(yyyymmdd);
}
while ((100 - freePercent) > maxPercent) {
String yyyymmdd = getLongAgoDate(deletedXLogDays);
if (yyyymmdd == null || today.equals(yyyymmdd)) {
break;
}
deleteData(yyyymmdd, false);
deleteData(yyyymmdd, Mode.XLOG);
usuableSpace = dbDir.getUsableSpace();
percent = (usuableSpace * 100.0d) / totalSpace;
freePercent = (usuableSpace * 100.0d) / totalSpace;
deletedXLogDays.add(yyyymmdd);
}
while ((100 - freePercent) > maxPercent) {
String yyyymmdd = getLongAgoDate(deletedDays);
if (yyyymmdd == null || today.equals(yyyymmdd)) {
break;
}
deleteData(yyyymmdd, Mode.ALL);
usuableSpace = dbDir.getUsableSpace();
freePercent = (usuableSpace * 100.0d) / totalSpace;
deletedDays.add(yyyymmdd);
}
}
}

int retainProfileDays = conf.mgr_purge_keep_days;
if (retainProfileDays > 0) {
if (today.equals(lastCheckDate) == false) {
lastCheckDate = today;
int lastDeleteDate = CastUtil.cint(DateUtil.yyyymmdd(System.currentTimeMillis() - (DateUtil.MILLIS_PER_DAY * retainProfileDays)));
while (true) {
String yyyymmdd = getLongAgoDate(deletedProfileDays);
if (yyyymmdd == null) {
break;
}
if (CastUtil.cint(yyyymmdd) > lastDeleteDate) {
break;
}
deleteData(yyyymmdd, Mode.PROFILE);
deletedProfileDays.add(yyyymmdd);
}
}
}
int retainDays = conf.mgr_purge_keep_days;
if (retainDays > 0) {

int retainXLogDays = conf.mgr_purge_xlog_without_profile_keep_days;
if (retainXLogDays > 0) {
if (today.equals(lastCheckDate) == false) {
lastCheckDate = today;
int lastDeleteDate = CastUtil.cint(DateUtil.yyyymmdd(System.currentTimeMillis() - (DateUtil.MILLIS_PER_DAY * retainDays)));
int lastDeleteDate = CastUtil.cint(DateUtil.yyyymmdd(System.currentTimeMillis() - (DateUtil.MILLIS_PER_DAY * retainXLogDays)));
while (true) {
String yyyymmdd = getLongAgoDate(deletedXLogDays);
if (yyyymmdd == null) {
Expand All @@ -104,17 +153,17 @@ public void run() {
if (CastUtil.cint(yyyymmdd) > lastDeleteDate) {
break;
}
deleteData(yyyymmdd, false);
deleteData(yyyymmdd, Mode.XLOG);
deletedXLogDays.add(yyyymmdd);
}
}
}

int retainNonXLogDays = conf.mgr_purge_non_xlog_keep_days;
if (retainNonXLogDays > 0) {
int retainCounterDays = conf.mgr_purge_counter_keep_days;
if (retainCounterDays > 0) {
if (today.equals(lastCheckDate) == false) {
lastCheckDate = today;
int lastDeleteDate = CastUtil.cint(DateUtil.yyyymmdd(System.currentTimeMillis() - (DateUtil.MILLIS_PER_DAY * retainNonXLogDays)));
int lastDeleteDate = CastUtil.cint(DateUtil.yyyymmdd(System.currentTimeMillis() - (DateUtil.MILLIS_PER_DAY * retainCounterDays)));
while (true) {
String yyyymmdd = getLongAgoDate(deletedDays);
if (yyyymmdd == null) {
Expand All @@ -123,7 +172,7 @@ public void run() {
if (CastUtil.cint(yyyymmdd) > lastDeleteDate) {
break;
}
deleteData(yyyymmdd, true);
deleteData(yyyymmdd, Mode.ALL);
deletedDays.add(yyyymmdd);
}
}
Expand All @@ -133,15 +182,26 @@ public void run() {
}
}

private void deleteData(String yyyymmdd, boolean all) {
private void deleteData(String yyyymmdd, Mode mode) {
try {
File f = null;
if (all) {
if (mode == Mode.ALL) {
f = new File(dbDir, yyyymmdd);
} else {
deleteFiles(f);
} else if(mode == Mode.XLOG) {
f = new File(dbDir, yyyymmdd + XLogWR.dir());
deleteFiles(f);
} else if (mode == Mode.PROFILE) {
f = new File(dbDir, yyyymmdd + XLogWR.dir() + "/xlog.profile");
deleteFiles(f);
f = new File(dbDir, yyyymmdd + XLogWR.dir() + "/xlog_profile.hfile");
deleteFiles(f);
f = new File(dbDir, yyyymmdd + XLogWR.dir() + "/xlog_profile.kfile");
deleteFiles(f);
} else {
throw new IllegalArgumentException("Not expected Mode : " + mode);
}
deleteFiles(f);

Logger.println("S206", "Auto deletion... " + yyyymmdd);
} catch (Throwable th) {
Logger.println("S207", "Failed auto deletion... " + yyyymmdd + " " + th.toString());
Expand Down

0 comments on commit 5f21867

Please sign in to comment.