Skip to content

Commit

Permalink
v1.5.1beta
Browse files Browse the repository at this point in the history
  • Loading branch information
@qjerome
qjerome committed May 14, 2019
1 parent 23ddbff commit 9e731e5
Show file tree
Hide file tree
Showing 9 changed files with 220 additions and 79 deletions.
13 changes: 7 additions & 6 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,12 +92,13 @@ which matched the event.
# Installation

## WHIDS
1. Run `install.bat` as administrator
2. Verify that files have been created at the **installation directory**
3. With a text editor **opened as administrator** open `config.json` and modify it as you wish
4. Skip this if running with a connection to a manager. If there is nothing in the **rules directory** the tool will be useless, so make sure there are some **gene** rules in there. You can get some compiled rules [here](https://raw.githubusercontent.com/0xrawsec/gene-rules/master/compiled.gen)
5. Start the HIDS with `Start.bat` script located in **installation directory**
6. If you configured a **manager** do not forget to run it
1. Download and extract the latest WHIDS release https://github.com/0xrawsec/whids/releases
2. Run `install.bat` as administrator
3. Verify that files have been created at the **installation directory**
4. With a text editor **opened as administrator** (to prevent changing rights of WHIDs installation directory) open `config.json` and modify it as you wish
5. Skip this if running with a connection to a manager. If there is nothing in the **rules directory** the tool will be useless, so make sure there are some **gene** rules in there. You can get some compiled rules [here](https://raw.githubusercontent.com/0xrawsec/gene-rules/master/compiled.gen)
6. Start the HIDS with `Start.bat` script located in **installation directory**
7. If you configured a **manager** do not forget to run it

**NB:** whenever you go to the installation directory with **File Explorer** and if you are **Administrator** the explorer will ask you if you want to change the permission of the directory. **DO NOT CLICK YES**, otherwise it will break the folder permissions put in place at installation time. Always access installation directory from **applications started as Administrator**.

Expand Down
125 changes: 89 additions & 36 deletions collector/forwarder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"time"

"github.com/0xrawsec/golang-evtx/evtx"
"github.com/0xrawsec/golang-utils/fileutils"
"github.com/0xrawsec/golang-utils/fsutil"
"github.com/0xrawsec/golang-utils/fsutil/fswalker"
"github.com/0xrawsec/golang-utils/fsutil/logfile"
Expand All @@ -24,6 +25,8 @@ const (
DefaultLogfileSize = logfile.MB * 5
// DiskSpaceThreshold allow 100MB of queued events
DiskSpaceThreshold = DefaultLogfileSize * 20
// MinRotationInterval is the minimum rotation interval allowed
MinRotationInterval = time.Minute
)

var ()
Expand All @@ -34,20 +37,36 @@ func buildURI(proto, host, port, url string) string {

}

// LoggingConfig structure to encode Logging configuration of the forwarder
type LoggingConfig struct {
Dir string `json:"dir"`
RotationInterval string `json:"rotation-interval"`
}

// ParseRotationInterval returns the parsed time.Duration
// from configuration structure.
func (l *LoggingConfig) ParseRotationInterval() (d time.Duration, err error) {
d, err = time.ParseDuration(l.RotationInterval)
if d < MinRotationInterval {
d = MinRotationInterval
}
return
}

// ForwarderConfig structure definition
type ForwarderConfig struct {
Client ClientConfig `json:"manager-client"`
LogsDir string `json:"logs-dir"`
Local bool `json:"local"`
Client ClientConfig `json:"manager-client"`
Logging LoggingConfig `json:"logging"`
Local bool `json:"local"`
}

// Forwarder structure definition
type Forwarder struct {
sync.Mutex
logsDir string
stop chan bool
done chan bool
logfile *logfile.LogFile
fwdConfig *ForwarderConfig
stop chan bool
done chan bool
logfile logfile.LogFile

Client *ManagerClient
TimeTresh time.Duration
Expand All @@ -63,7 +82,9 @@ func NewForwarder(c *ForwarderConfig) (*Forwarder, error) {
var err error

// Initialize the Forwarder
// TODO: better organize forwarder configuration
co := Forwarder{
fwdConfig: c,
TimeTresh: time.Second * 10,
EventTresh: 50,
Pipe: new(bytes.Buffer),
Expand All @@ -79,15 +100,15 @@ func NewForwarder(c *ForwarderConfig) (*Forwarder, error) {
}

// queue directory
co.logsDir = c.LogsDir
if c.LogsDir == "" {
c.Logging.Dir = c.Logging.Dir
if c.Logging.Dir == "" {
return nil, fmt.Errorf("Field \"logs-dir\" is missing from configuration")
}

// creating the queue directory
if !fsutil.Exists(co.logsDir) && !fsutil.IsDir(co.logsDir) {
if !fsutil.Exists(c.Logging.Dir) && !fsutil.IsDir(c.Logging.Dir) {
// TOCTU may happen here so we double check error code
if err = os.Mkdir(co.logsDir, DefaultDirPerm); err != nil && !os.IsExist(err) {
if err = os.Mkdir(c.Logging.Dir, DefaultDirPerm); err != nil && !os.IsExist(err) {
return nil, fmt.Errorf("Cannot create queue directory : %s", err)
}
}
Expand All @@ -103,6 +124,25 @@ func (f *Forwarder) LogfilePath() string {
return ""
}

// ArchiveLogs archives the old log files not compressed into compressed
func (f *Forwarder) ArchiveLogs() {
for wi := range fswalker.Walk(f.fwdConfig.Logging.Dir) {
for _, fi := range wi.Files {
// fullpath
fp := filepath.Join(f.fwdConfig.Logging.Dir, fi.Name())
log.Infof("Archiving old log: %s", fp)

if !strings.HasSuffix(fp, ".gz") {
if err := fileutils.GzipFile(fp); err != nil {
log.Errorf("Failed to archive log: %s", err)

}
}
}
}

}

// PipeEvent pipes an event to be sent through the forwarder
func (f *Forwarder) PipeEvent(e *evtx.GoEvtxMap) {
f.Lock()
Expand All @@ -114,7 +154,6 @@ func (f *Forwarder) PipeEvent(e *evtx.GoEvtxMap) {

// Save save the piped events to the disks
func (f *Forwarder) Save() error {
var err error
log.Infof("Collector saved logs to be sent later on")

// Clean queued files if needed
Expand All @@ -128,18 +167,24 @@ func (f *Forwarder) Save() error {

if f.logfile == nil {
// This will reopen the first available alerts.gz.X file if several
lf := filepath.Join(f.logsDir, "alerts.gz")
if f.logfile, err = logfile.OpenFile(lf, DefaultLogPerm, DefaultLogfileSize); err != nil {
//lf := filepath.Join(f.fwdConfig.LogConf.Dir, "alerts.gz")
lf := filepath.Join(f.fwdConfig.Logging.Dir, "alerts")
ri, err := f.fwdConfig.Logging.ParseRotationInterval()
if err != nil {
return err
}
log.Infof("Rotating logfile every %s", ri)
if f.logfile, err = logfile.OpenTimeRotateLogFile(lf, DefaultLogPerm, ri, time.Second*5); err != nil {
return err
}
}
f.logfile.Write(f.Pipe.Bytes())
return nil
_, err := f.logfile.Write(f.Pipe.Bytes())
return err
}

// HasQueuedEvents checks whether some events are waiting to be sent
func (f *Forwarder) HasQueuedEvents() bool {
for wi := range fswalker.Walk(f.logsDir) {
for wi := range fswalker.Walk(f.fwdConfig.Logging.Dir) {
if len(wi.Files) > 0 {
return true
}
Expand All @@ -152,18 +197,18 @@ func (f *Forwarder) CleanOlderQueued() error {
var older string
var olderTime time.Time
empty := time.Time{}
for wi := range fswalker.Walk(f.logsDir) {
for wi := range fswalker.Walk(f.fwdConfig.Logging.Dir) {
for _, fi := range wi.Files {
// initialization
if olderTime == empty {
olderTime = fi.ModTime()
older = filepath.Join(f.logsDir, fi.Name())
older = filepath.Join(f.fwdConfig.Logging.Dir, fi.Name())
continue
}
// check if we have an older file
if fi.ModTime().Before(olderTime) {
olderTime = fi.ModTime()
older = filepath.Join(f.logsDir, fi.Name())
older = filepath.Join(f.fwdConfig.Logging.Dir, fi.Name())
}
}
}
Expand All @@ -180,7 +225,7 @@ func (f *Forwarder) CleanOlderQueued() error {
// DiskSpaceQueue compute the disk space (in bytes) taken by queued events
func (f *Forwarder) DiskSpaceQueue() int64 {
var dp int64
for wi := range fswalker.Walk(f.logsDir) {
for wi := range fswalker.Walk(f.fwdConfig.Logging.Dir) {
for _, fi := range wi.Files {
dp += fi.Size()
}
Expand All @@ -199,6 +244,12 @@ func (f *Forwarder) ProcessQueue() {
return
}

// returns if Manager is not up, this prevents closing logfile
// for nothing
if !f.Client.IsServerUp() {
return
}

log.Info("Processing queued files")

if f.logfile != nil {
Expand All @@ -207,32 +258,32 @@ func (f *Forwarder) ProcessQueue() {

// Reset logfile for latter Save function use
f.logfile = nil
for wi := range fswalker.Walk(f.logsDir) {
for wi := range fswalker.Walk(f.fwdConfig.Logging.Dir) {
for _, fi := range wi.Files {
// fullpath
fp := filepath.Join(f.logsDir, fi.Name())
fp := filepath.Join(f.fwdConfig.Logging.Dir, fi.Name())
log.Debug("Processing queued file: %s", fp)
fd, err := os.Open(fp)
if err != nil {
log.Errorf("Failed to open queued file (%s): %s", fp, err)
continue
}

// the file is gzip so we have to pass a gzip reader to prepCollectReq
gzr, err := gzip.NewReader(fd)
if err != nil {
log.Errorf("Failed to create gzip reader for queued file (%s): %s", fp, err)
// close file
if strings.HasSuffix(fp, ".gz") {
// the file is gzip so we have to pass a gzip reader to prepCollectReq
gzr, err := gzip.NewReader(fd)
if err != nil {
log.Errorf("Failed to create gzip reader for queued file (%s): %s", fp, err)
// close file
fd.Close()
continue
}
err = f.Client.PostLogs(gzr)
// we can close the reader since we don't need those anymore
gzr.Close()
fd.Close()
continue
}

err = f.Client.PostLogs(gzr)

// we can close the reader since we don't need those anymore
gzr.Close()
fd.Close()

// We do not remove the logs if we failed to send
if err != nil {
log.Errorf("%s", err)
Expand Down Expand Up @@ -296,11 +347,13 @@ func (f *Forwarder) Run() {
default:
}
// We have queued events so we try to send them before sending pending events
// We check if server is up not to close the current logfile if not needed
if f.HasQueuedEvents() {
f.ProcessQueue()
}

// Sending piped events
if f.EventsPiped >= f.EventTresh || time.Now().After(timer.Add(f.TimeTresh)) {
if f.EventsPiped >= f.EventTresh || time.Now().After(timer.Add(f.TimeTresh)) || f.Local {
// Send out events if there are pending events
if f.EventsPiped > 0 {
f.Collect()
Expand Down
8 changes: 4 additions & 4 deletions collector/manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ type Manager struct {
rulesDir string
containersDir string
authorized datastructs.SyncedSet
logfile *logfile.LogFile
logfile logfile.LogFile
tls TLSConfig
srv *http.Server
stop chan bool
Expand All @@ -197,7 +197,7 @@ func NewManager(c *ManagerConfig) (*Manager, error) {
}

m := Manager{Host: c.Host, Port: fmt.Sprintf("%d", c.Port)}
m.logfile, err = logfile.OpenFile(c.Logfile, DefaultLogPerm, DefaultManagerLogSize)
m.logfile, err = logfile.OpenTimeRotateLogFile(c.Logfile, DefaultLogPerm, time.Hour, time.Second*5)
m.key = c.Key
m.authorized = datastructs.NewInitSyncedSet(datastructs.ToInterfaceSlice(c.Authorized)...)
m.stop = make(chan bool)
Expand Down Expand Up @@ -491,11 +491,11 @@ func (m *Manager) Collect(wt http.ResponseWriter, rq *http.Request) {
default:
// Todo put there code to validate that logs is JSON format
log.Debugf("Received Event: %s", tok)
m.logfile.WriteString(fmt.Sprintln(tok))
m.logfile.Write([]byte(fmt.Sprintln(tok)))
cnt++
}
}
// force logfile to flush events to disk
m.logfile.Flush()
//m.logfile.Flush()
log.Debugf("Count Event Received: %d", cnt)
}
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ module github.com/0xrawsec/whids
require (
github.com/0xrawsec/gene v1.4.8
github.com/0xrawsec/golang-evtx v1.2.0
github.com/0xrawsec/golang-utils v1.1.1
github.com/0xrawsec/golang-utils v1.1.2
github.com/0xrawsec/golang-win32 v1.0.0
github.com/0xrawsec/mux v1.6.2
golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ github.com/0xrawsec/golang-utils v1.1.0 h1:opQAwRONEfxOOl4nxhpPkXiTYgzAw0/wFATAf
github.com/0xrawsec/golang-utils v1.1.0/go.mod h1:DADTtCFY10qXjWmUVhhJqQIZdSweaHH4soYUDEi8mj0=
github.com/0xrawsec/golang-utils v1.1.1 h1:HlwVs5lHl5rK2DhB1eDlf+J9hOKBHEObQCWXFcQ4GE0=
github.com/0xrawsec/golang-utils v1.1.1/go.mod h1:DADTtCFY10qXjWmUVhhJqQIZdSweaHH4soYUDEi8mj0=
github.com/0xrawsec/golang-utils v1.1.2 h1:7XHIpeCltIf+s70JOnYCMEIVsthZwRGlu6041J9tE9o=
github.com/0xrawsec/golang-utils v1.1.2/go.mod h1:DADTtCFY10qXjWmUVhhJqQIZdSweaHH4soYUDEi8mj0=
github.com/0xrawsec/golang-win32 v1.0.0 h1:vxraJixIhyq63Y4h/58j/m6X5iT2LxonTTllz64pP4E=
github.com/0xrawsec/golang-win32 v1.0.0/go.mod h1:jSzAgD20VWKep8qW89ZjgAx0NcIjfPSDteYDoNqSZ2g=
github.com/0xrawsec/mux v1.6.2 h1:cc2OyJTxRmXxsmQe2ulp0VndXV8vZIRrc1JqQzJ4BMI=
Expand Down
Loading

0 comments on commit 9e731e5

Please sign in to comment.