Skip to content

Commit

Permalink
improve alerta server correlation logic
Browse files Browse the repository at this point in the history
  • Loading branch information
@satterly
satterly committed Oct 2, 2013
1 parent 623c38e commit 3aa5dfa
Show file tree
Hide file tree
Showing 5 changed files with 39 additions and 26 deletions.
2 changes: 1 addition & 1 deletion alerta/common/alert.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@

class Alert(object):

def __init__(self, resource, event, correlate=None, group='Misc', value=None, status=status_code.OPEN,
def __init__(self, resource, event, correlate=None, group='Misc', value=None, status=status_code.UNKNOWN,
severity=severity_code.NORMAL, previous_severity=severity_code.UNKNOWN, environment=None, service=None,
text=None, event_type='exceptionAlert', tags=None, origin=None, repeat=False, duplicate_count=0,
threshold_info='n/a', summary=None, timeout=None, alertid=None, last_receive_id=None,
Expand Down
1 change: 1 addition & 0 deletions alerta/common/status_code.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
ACK: ACK_STATUS_CODE,
CLOSED: CLOSED_STATUS_CODE,
EXPIRED: EXPIRED_STATUS_CODE,
UNKNOWN: UNKNOWN_STATUS_CODE,
}


Expand Down
22 changes: 14 additions & 8 deletions alerta/server/daemon.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,10 +73,15 @@ def run(self):
# Duplicate alert .. 1. update existing document with lastReceiveTime, lastReceiveId, text, summary,
# value, status, tags and origin
# 2. increment duplicate count
# 3. update and push status if changed

LOG.info('%s : Duplicate alert -> update dup count', incomingAlert.alertid)
duplicateAlert = self.db.duplicate_alert(incomingAlert)

if incomingAlert.status != status_code.UNKNOWN and incomingAlert.status != duplicateAlert.status:
self.db.update_status(alert=duplicateAlert, status=incomingAlert.status)
duplicateAlert.status = incomingAlert.status

if CONF.forward_duplicate:
# Forward alert to notify topic and logger queue
self.mq.send(duplicateAlert, CONF.outbound_queue)
Expand All @@ -91,20 +96,19 @@ def run(self):
# lastReceiveTime, previousSeverity,
# severityCode, lastReceiveId, text, summary, value, tags and origin
# 2. set duplicate count to zero
# 3. push history
# 3. push history and status if changed

previous_severity = self.db.get_severity(incomingAlert)
LOG.info('%s : Event and/or severity change %s %s -> %s update details', incomingAlert.get_id(),
incomingAlert.event, previous_severity, incomingAlert.severity)

trend_indication = severity_code.trend(previous_severity, incomingAlert.severity)

correlatedAlert = self.db.update_alert(incomingAlert, previous_severity, trend_indication)
correlatedAlert = self.db.correlate_alert(incomingAlert, previous_severity, trend_indication)

status = severity_code.status_from_severity(previous_severity, correlatedAlert.severity)
if status:
self.db.update_status(alert=correlatedAlert, status=status)
correlatedAlert.status = status
if incomingAlert.status != status_code.UNKNOWN and incomingAlert.status != correlatedAlert.status:
self.db.update_status(alert=correlatedAlert, status=incomingAlert.status)
correlatedAlert.status = incomingAlert.status

# Forward alert to notify topic and logger queue
self.mq.send(correlatedAlert, CONF.outbound_queue)
Expand All @@ -116,7 +120,7 @@ def run(self):
else:
LOG.info('%s : New alert -> insert', incomingAlert.get_id())
# New alert so ... 1. insert entire document
# 2. push history
# 2. push history and status
# 3. set duplicate count to zero

trend_indication = severity_code.trend(severity_code.UNKNOWN, incomingAlert.severity)
Expand All @@ -127,12 +131,14 @@ def run(self):
incomingAlert.last_receive_time = incomingAlert.receive_time
incomingAlert.trend_indication = trend_indication

if incomingAlert.status == status_code.OPEN:
if incomingAlert.status == status_code.UNKNOWN:
incomingAlert.status = severity_code.status_from_severity(severity_code.UNKNOWN, incomingAlert.severity)

if incomingAlert.alertid != self.db.save_alert(incomingAlert):
LOG.critical('Alert was not saved with submitted alert id. Race condition?')

self.db.update_status(alert=incomingAlert, status=incomingAlert.status)

# Forward alert to notify topic and logger queue
self.mq.send(incomingAlert, CONF.outbound_queue)
self.mq.send(incomingAlert, CONF.outbound_topic)
Expand Down
36 changes: 21 additions & 15 deletions alerta/server/database.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -194,30 +194,36 @@ def get_alert(self, alertid=None, environment=None, resource=None, event=None, s
history=response.get('history', None),
)

def update_alert(self, alert, previous_severity=None, trend_indication=None):
def correlate_alert(self, alert, previous_severity=None, trend_indication=None):

previous_severity = previous_severity or severity_code.UNKNOWN
trend_indication = trend_indication or severity_code.NO_CHANGE

update = {
"event": alert.event,
"correlatedEvents": alert.correlate,
"group": alert.group,
"value": alert.value,
"severity": alert.severity,
"createTime": alert.create_time,
"receiveTime": alert.receive_time,
"lastReceiveTime": alert.receive_time,
"expireTime": alert.expire_time,
"previousSeverity": previous_severity,
"lastReceiveId": alert.alertid,
"service": alert.service,
"text": alert.text,
"summary": alert.summary,
"value": alert.value,
"tags": alert.tags,
"repeat": False,
"origin": alert.origin,
"repeat": False,
"duplicateCount": 0,
"thresholdInfo": alert.threshold_info,
"summary": alert.summary,
"timeout": alert.timeout,
"lastReceiveId": alert.alertid,
"createTime": alert.create_time,
"expireTime": alert.expire_time,
"receiveTime": alert.receive_time,
"lastReceiveTime": alert.receive_time,
"trendIndication": trend_indication,
"rawData": alert.raw_data,
"duplicateCount": 0,
"moreInfo": alert.more_info,
"graphUrls": alert.graph_urls,
}

query = {"environment": alert.environment, "resource": alert.resource,
Expand All @@ -231,13 +237,13 @@ def update_alert(self, alert, previous_severity=None, trend_indication=None):
update={'$set': update,
'$push': {
"history": {
"createTime": update['createTime'],
"receiveTime": update['receiveTime'],
"severity": update['severity'],
"id": update['lastReceiveId'],
"event": update['event'],
"severity": update['severity'],
"value": update['value'],
"text": update['text'],
"id": update['lastReceiveId']
"createTime": update['createTime'],
"receiveTime": update['receiveTime'],
}
}
},
Expand Down Expand Up @@ -421,9 +427,9 @@ def duplicate_alert(self, alert):
"correlatedEvents": alert.correlate,
"group": alert.group,
"value": alert.value,
"status": alert.status,
"service": alert.service,
"text": alert.text,
"tags": alert.tags,
"origin": alert.origin,
"repeat": True,
"thresholdInfo": alert.threshold_info,
Expand Down
4 changes: 2 additions & 2 deletions bin/alert-sender
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,8 @@ def main(argv):
)
parser.add_argument(
'--status',
default='Open',
help='Status eg. Open, Ack, Closed (default: %(default)s)'
default='Unknown',
help='Status eg. Open, Ack, Closed, Unknown (default: %(default)s)'
)
parser.add_argument(
'-s',
Expand Down

0 comments on commit 3aa5dfa

Please sign in to comment.