Skip to content

Commit

Permalink
Merge pull request #13 from FTWynn/log_reconversion
Browse files Browse the repository at this point in the history 
Replace Logging Library
  • Loading branch information
@FTWynn
FTWynn committed Feb 4, 2016
2 parents 8a40702 + c0674cb commit 1874211
Show file tree
Hide file tree
Showing 5 changed files with 226 additions and 118 deletions.
2 changes: 1 addition & 1 deletion .travis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,6 @@ go:
- tip

install:
- go get gopkg.in/inconshreveable/log15.v2
- go get github.com/Sirupsen/logrus
- go get github.com/ftwynn/gologgen/loggensender
- go get github.com/ftwynn/gologgen/loggenmunger
74 changes: 46 additions & 28 deletions loggenmunger/loggenmunger.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,38 +7,40 @@ import (
"strings"
"time"

"github.com/ftwynn/gologgen/loghelper"

log15 "gopkg.in/inconshreveable/log15.v2"
log "github.com/Sirupsen/logrus"
)

var log log15.Logger

func init() {
log15.Root().SetHandler(log15.LvlFilterHandler(log15.LvlInfo, log15.StdoutHandler))
log = log15.New("function", log15.Lazy{Fn: loghelper.Log15LazyFunctionName})
}

// RandomizeString takes a string, looks for the random tokens (int, string, and timestamp), and replaces them
// RandomizeString takes a string, looks for the random tokens
// (int, string, and timestamp), and replaces them
func RandomizeString(text string, timeformat string) string {
log.Debug("Starting String Randomization", "text", text, "timeFormat", timeformat)
log.WithFields(log.Fields{
"text": text,
"timeformat": timeformat,
}).Debug("Starting String Randomization")

// Bail if we can't get any randomizers
goodstring, err := regexp.MatchString(`\$\[[^\]]+\]`, text)
if err != nil {
log.Error("Something broke on parsing the text string with a regular expression", "error_msg", err, "text", text, "regex", `\$\[[^\]]+\]`)
log.WithFields(log.Fields{
"error_msg": err,
"text": text,
"regex": `\$\[[^\]]+\]`,
}).Error("Something broke on parsing the text string with a regular expression")
}

//Return original string if 0 randomizers
if !goodstring {
log.Debug("Found no random tokens: ", "text", text)
log.Debug("Found no random tokens, returning the original string")
return text
}

// Find all randomizing tokens
re := regexp.MustCompile(`\$\[[^\]]+\]`)
randos := re.FindAllString(text, -1)
log.Debug("A found random tokens", "num", len(randos), "randomTokens", randos)
log.WithFields(log.Fields{
"num": len(randos),
"randomTokens": randos,
}).Debug("Found random tokens")

// Create a list of new strings to be inserted where the tokens were
var newstrings []string
Expand All @@ -48,18 +50,30 @@ func RandomizeString(text string, timeformat string) string {
for _, rando := range randos {
// Take off the leading and trailing formatting
tempstring := replacer.Replace(rando)
log.Debug("Removing the formatting from the items: ", "tempstring", tempstring)
log.WithFields(log.Fields{
"tempstring": tempstring,
}).Debug("Removing the formatting from the items")

// Split the randomizer into individual items
tempstrings := strings.Split(tempstring, ",")
log.Debug("Splitting the random tokens up: ", "tempstrings", tempstrings, "count", len(tempstrings))
log.WithFields(log.Fields{
"tempstrings": tempstrings,
"count": len(tempstrings),
}).Debug("Splitting the random tokens up")

// Numeric ranges will only have two items for an upper and lower bound, timestamps have "time" and "stamp", all the rest are string groups
// Numeric ranges will only have two items for an upper and lower bound,
// timestamps have "time" and "stamp", all the rest are string groups
var randType string
num0, err := strconv.Atoi(string(tempstrings[0]))
num1, err2 := strconv.Atoi(string(tempstrings[1]))
log.Debug("Parsing entry 0 as a number: ", "num", num0, "error", err)
log.Debug("Parsing entry 1 as a number: ", "num", num1, "error", err2)
log.WithFields(log.Fields{
"num": num0,
"error_msg": err,
}).Debug("Parsing entry 0 as a number")
log.WithFields(log.Fields{
"num": num1,
"error_msg": err2,
}).Debug("Parsing entry 1 as a number", "num", num1, "error", err2)

switch {
case len(tempstrings) == 2 && err == nil && err2 == nil:
Expand All @@ -70,24 +84,28 @@ func RandomizeString(text string, timeformat string) string {
randType = "Category"
}

log.Debug("What type of token is this?", "type", randType)
log.WithFields(log.Fields{
"type": randType,
}).Debug("Finished determining token type")

switch randType {
case "Category":
newstrings = append(newstrings, tempstrings[rand.Intn(len(tempstrings))])
case "Number":
// Get a random number in the range
diff := num1 - num0
log.Debug("Difference from second and first numbers", "diff", diff)
log.Debug("Difference from second and first numbers: ", "diff - ", diff)
tempnum := rand.Intn(diff)
log.Debug("Random number from zero adjusted spread", "rand", tempnum)
log.Debug("Random number adjusted to range and string converted", "rand", strconv.Itoa(tempnum+num0))
log.Debug("Random number from zero adjusted spread: ", "rand - ", tempnum)
log.Debug("Random number adjusted to range and string converted: ", "rand - ", strconv.Itoa(tempnum+num0))
newstrings = append(newstrings, strconv.Itoa(tempnum+num0))
case "Timestamp":
t := time.Now()
timeformatted, err := formatTimestamp(t, timeformat)
if err != nil {
log.Error("Formatting the timestamp broke")
log.WithFields(log.Fields{
"error_msg": err,
}).Error("Formatting the timestamp broke")
}
newstrings = append(newstrings, timeformatted)
}
Expand All @@ -103,13 +121,13 @@ func RandomizeString(text string, timeformat string) string {
}
}

log.Debug("Randomization complete", "newString", strings.Join(newLogLine, ""))
log.Debug("Randomization complete: ", "newString - ", strings.Join(newLogLine, ""))

return strings.Join(newLogLine, "")
}

func formatTimestamp(t time.Time, timeformat string) (string, error) {
log.Debug("Current time", "now", t)
log.Debug("Current time: ", "now - ", t)
var timeformatted string
switch timeformat {
case "epoch":
Expand All @@ -121,6 +139,6 @@ func formatTimestamp(t time.Time, timeformat string) (string, error) {
default:
timeformatted = t.Format(timeformat)
}
log.Debug("Formatted time", "now", timeformatted)
log.Debug("Formatted time: ", "now - ", timeformatted)
return timeformatted, nil
}
97 changes: 67 additions & 30 deletions loggensender/loggensender.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,20 +10,17 @@ import (
"time"

"github.com/ftwynn/gologgen/loggenmunger"
"github.com/ftwynn/gologgen/loghelper"

log15 "gopkg.in/inconshreveable/log15.v2"
log "github.com/Sirupsen/logrus"
)

var log log15.Logger

// LogLineProperties holds all the data relevant to running a Log Line
type LogLineProperties struct {
OutputType string
SyslogType string
SyslogLoc string
HTTPLoc string `json:"HTTPLoc"`
PostBody string `json:"Text"`
Text string `json:"Text"`
IntervalSecs int `json:"IntervalSecs"`
IntervalStdDev float64 `json:"IntervalStdDev"`
TimestampFormat string `json:"TimestampFormat"`
Expand All @@ -39,18 +36,24 @@ type LogLineHTTPHeader struct {
Value string `json:"Value"`
}

// I'm not really sure why this bit is required (and doesn't overwrite what's in main)... I may need to build my own logging library so I can grasp all the particulars
func init() {
log15.Root().SetHandler(log15.LvlFilterHandler(log15.LvlError, log15.StdoutHandler))
log = log15.New("function", log15.Lazy{Fn: loghelper.Log15LazyFunctionName})
}

// DispatchLogs takes a slice of Log Lines and a time and fires the ones listed, re-adding them to the Run Table where the next run should go
func DispatchLogs(RunTable *map[time.Time][]LogLineProperties, ThisTime time.Time) {

log.Info("Starting Dispatch Logs")
RunTableObj := *RunTable
log.Info("Starting Dispatch Logs", "time", ThisTime, "length", len(RunTableObj[ThisTime]))

// If no log lines, clean up and exit
if len(RunTableObj[ThisTime]) == 0 {
delete(RunTableObj, ThisTime)
log.WithFields(log.Fields{
"time": ThisTime,
}).Info("No logs to dispatch, exiting")
return
}

log.WithFields(log.Fields{
"time": ThisTime,
"count": len(RunTableObj[ThisTime]),
}).Info("Starting Dispatch Logs")

// get a rand object for later
r := rand.New(rand.NewSource(time.Now().UnixNano()))
Expand All @@ -68,21 +71,25 @@ func DispatchLogs(RunTable *map[time.Time][]LogLineProperties, ThisTime time.Tim
nextInterval = 1000
}
nextTime := ThisTime.Add(time.Duration(nextInterval) * time.Millisecond).Truncate(time.Second)
log.Info("SCHEDULED - Next log run", "line", line.PostBody, "nextTime", nextTime)
log.WithFields(log.Fields{
"line": line.Text,
"nextTime": nextTime,
}).Info("SCHEDULED - Next log run")
RunTableObj[nextTime] = append(RunTableObj[nextTime], line)

}

delete(RunTableObj, ThisTime)
log.Info("Finished dispatching logs", "time", ThisTime)
log.WithFields(log.Fields{
"time": ThisTime,
}).Info("Finished dispatching logs")
}

// RunLogLine runs an instance of a log line through the appropriate output
func RunLogLine(params LogLineProperties, sendTime time.Time) {
log.Info("Starting Individual Log Runner", "time", sendTime, "logline", params.PostBody)

// Randomize the post body if need be
var stringBody = []byte(loggenmunger.RandomizeString(params.PostBody, params.TimestampFormat))
// Randomize the text if need be
var stringBody = []byte(loggenmunger.RandomizeString(params.Text, params.TimestampFormat))

switch params.OutputType {
case "http":
Expand All @@ -92,46 +99,73 @@ func RunLogLine(params LogLineProperties, sendTime time.Time) {
case "file":
go sendLogLineFile(stringBody, params)
}
log.Info("Finished Individual Log Runner", "time", sendTime, "logline", params.PostBody)
}

// sendLogLineHTTP sends the log line to the http endpoint, retrying if need be
func sendLogLineHTTP(client *http.Client, stringBody []byte, params LogLineProperties) {
// Post to Sumo
log.Info("Sending log to Sumo over HTTP", "line", string(stringBody))
// Post to HTTP
log.WithFields(log.Fields{
"line": string(stringBody),
}).Info("Sending log over HTTP")

req, err := http.NewRequest("POST", params.HTTPLoc, bytes.NewBuffer(stringBody))
for _, header := range params.Headers {
req.Header.Add(header.Header, header.Value)
}
log.Debug("Request object to send to Sumo", "request", req)
log.WithFields(log.Fields{
"request": req,
}).Debug("Request object to send to Sumo")

resp, err := client.Do(req)
defer resp.Body.Close()
if err != nil {
log.Error("Something went amiss on submitting to Sumo", "error_msg", err, "line", string(stringBody))
log.WithFields(log.Fields{
"error_msg": err,
"line": string(stringBody),
}).Error("Something went wrong with the http client")
return
}

// For non 200 StatusCode, retry 5 times and then give up
if resp.StatusCode != 200 {
log.Debug("Non 200 response, retrying")
for i := 0; i < 5; i++ {
log.Debug("Retrying", "attemptNumber", i+1)
log.WithFields(log.Fields{
"attemptNumber": i + 1,
}).Debug("Retrying HTTP Post")
resp2, err := client.Do(req)
defer resp.Body.Close()
if resp2.StatusCode == 200 && err == nil {
break
}
if i == 4 {
log.WithFields(log.Fields{
"error_msg": err,
"line": string(stringBody),
}).Error("Got non-200 response from HTTP Location and retries failed")
}
time.Sleep(time.Duration(10) * time.Second)
}
}
log.Debug("Response from Sumo", "statusCode", resp.StatusCode)
log.WithFields(log.Fields{
"statusCode": resp.StatusCode,
}).Debug("Response from Sumo")
}

//sendLogLineSyslog sends the log on tcp/udp, WITHOUT retrying
func sendLogLineSyslog(stringBody []byte, params LogLineProperties) {
log.Info("Sending log to syslog", "line", string(stringBody), "location", params.SyslogLoc)
log.WithFields(log.Fields{
"line": string(stringBody),
"location": params.SyslogLoc,
}).Info("Sending log to syslog")

conn, err := net.Dial(params.SyslogType, params.SyslogLoc)
if err != nil {
log.Error("Failed to create syslog connection, abandoning", "error_msg", err, "type", params.SyslogType, "syslogLocation", params.SyslogLoc)
log.WithFields(log.Fields{
"error_msg": err,
"type": params.SyslogType,
"syslogLocation": params.SyslogLoc,
}).Error("Failed to create syslog connection, abandoning")
}
defer conn.Close()

Expand All @@ -140,12 +174,15 @@ func sendLogLineSyslog(stringBody []byte, params LogLineProperties) {

//sendLogLineFile writes log lines to a file
func sendLogLineFile(stringBody []byte, params LogLineProperties) {
log.Info("Writing log to file", "line", string(stringBody))
log.WithFields(log.Fields{
"line": string(stringBody),
}).Info("Writing log to file")

_, err := params.FileHandler.Write(append(stringBody, []byte("\n")...))
if err != nil {
log.Error("Error writing to file", "error_msg", err)
panic(err)
log.WithFields(log.Fields{
"error_msg": err,
}).Fatal("Error writing to file")
}

}
13 changes: 0 additions & 13 deletions loghelper/loghelper.go
View file

This file was deleted.

Loading

0 comments on commit 1874211

Please sign in to comment.