Skip to content

Commit

Permalink
coverage improvement and bug tracking session
Browse files Browse the repository at this point in the history
  • Loading branch information
@qjerome
qjerome committed Sep 2, 2022
1 parent a67d5cf commit e78e9c8
Show file tree
Hide file tree
Showing 21 changed files with 494 additions and 230 deletions.
2 changes: 1 addition & 1 deletion .github/coverage/badge.svg
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
199 changes: 100 additions & 99 deletions .github/coverage/coverage.txt
View file

Large diffs are not rendered by default.

6 changes: 4 additions & 2 deletions agent/agent.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,9 @@ func (a *Agent) Prepare(c *config.Agent) (err error) {
a.actionHandler = NewActionHandler(a)

// Creates missing directories
c.Prepare()
if err = c.Prepare(); err != nil {
return
}

// Create logfile asap if needed
if c.Logfile != "" {
Expand Down Expand Up @@ -827,7 +829,7 @@ func (a *Agent) Run() {
a.scheduler.Start()

for _, t := range a.scheduler.Tasks() {
a.logger.Infof("Scheduler running: %s", t.Name)
a.logger.Infof("Scheduler running: %s (async=%t) (scheduled=%t)", t.Name, t.IsAsync(), t.IsScheduled())
}

// Dry run don't do anything
Expand Down
31 changes: 30 additions & 1 deletion agent/agent_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,15 @@ func init() {

}

func testingRule() (r engine.Rule) {
r = engine.NewRule()
r.Name = "Testing:MatchAllSysmon"
// FileCreate, FileDeleted and FileDeletedDetected
r.Meta.Events = map[string][]int64{"Microsoft-Windows-Sysmon/Operational": {}}
r.Meta.Criticality = 10
return r
}

func generateCert(c server.ManagerConfig) {
hosts := []string{c.AdminAPI.Host, c.EndpointAPI.Host}
key, cert, err := utils.GenerateCert("Test", hosts, time.Hour*24*365)
Expand Down Expand Up @@ -269,6 +278,7 @@ func TestAgent(t *testing.T) {
installSysmon()

var gotSysmonEvent bool
var gotProcessTermination bool

tmp, err := utils.HidsMkTmpDir()
tt.CheckErr(err)
Expand All @@ -279,6 +289,10 @@ func TestAgent(t *testing.T) {
c.Logfile = ""
c.FwdConfig.Local = false
c.FwdConfig.Client = clConf
// enable audit policy to trigger FileSystem events hooks
c.AuditConfig.Enable = true
c.AuditConfig.AuditDirs = []string{`C:\Windows`, `C:\Users`}
// empty all actions
c.Actions = config.Actions{
AvailableActions: AvailableActions,
Low: []string{},
Expand All @@ -287,7 +301,14 @@ func TestAgent(t *testing.T) {
Critical: []string{},
}

// creating new agent
a, err := NewAgent(c)
tt.CheckErr(err)

// loading testing rule
r := testingRule()
tt.CheckErr(a.Engine.LoadRule(&r))

a.logger.ErrorHandler = tt.CheckErr
// reduce scheduled task ticker
for _, t := range a.scheduler.Tasks() {
Expand All @@ -301,6 +322,9 @@ func TestAgent(t *testing.T) {
if e.Channel() == sysmonChannel {
gotSysmonEvent = true
}
if isSysmonProcessTerminate(e) {
gotProcessTermination = true
}
// create fake detection to cover action
d := engine.NewDetection(true, true)
// enable all actions
Expand All @@ -321,8 +345,13 @@ func TestAgent(t *testing.T) {
a.Stop()

tt.Assert(gotSysmonEvent, "failed to monitor Sysmon events")
tt.Assert(gotProcessTermination, "failed to get Sysmon process termination event")

t.Log(utils.PrettyJsonOrPanic(a.Report(false)))
report := a.Report(false)
for _, c := range report.Commands {
// control that we did not get any error
tt.Assert(c.Error == "")
}

a.WaitWithTimeout(time.Second * 15)

Expand Down
108 changes: 59 additions & 49 deletions agent/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (
"github.com/0xrawsec/golang-utils/fsutil"
"github.com/0xrawsec/whids/api/client/config"
"github.com/0xrawsec/whids/utils"
"github.com/pelletier/go-toml"
"github.com/pelletier/go-toml/v2"
)

const (
Expand All @@ -20,38 +20,34 @@ const (
ActionCriticalLow, ActionCriticalHigh = 10, 10
)

var (
emptyForwarderConfig = config.Forwarder{}
)

type Actions struct {
AvailableActions []string `json:"available-actions" toml:"available-actions" commented:"true" comment:"List of available actions (here as a memo for easier configuration, but it is not used in any way by the engine)"`
Low []string `json:"low" toml:"low" comment:"Default actions to be taken when event criticality is in [1; 4]"`
Medium []string `json:"medium" toml:"medium" comment:"Default actions to be taken when event criticality is in [5; 7]"`
High []string `json:"high" toml:"high" comment:"Default actions to be taken when event criticality is in [8; 9]"`
Critical []string `json:"critical" toml:"critical" comment:"Default actions to be taken when event criticality is 10"`
AvailableActions []string `json:"available-actions,omitempty" toml:"available-actions" comment:"List of available actions (here as a memo for easier configuration, but it is not used in any way by the engine)"`
Low []string `json:"low,omitempty" toml:"low" comment:"Default actions to be taken when event criticality is in [1; 4]"`
Medium []string `json:"medium,omitempty" toml:"medium" comment:"Default actions to be taken when event criticality is in [5; 7]"`
High []string `json:"high,omitempty" toml:"high" comment:"Default actions to be taken when event criticality is in [8; 9]"`
Critical []string `json:"critical,omitempty" toml:"critical" comment:"Default actions to be taken when event criticality is 10"`
}

// Dump structure definition
type Dump struct {
Dir string `json:"dir" toml:"dir" comment:"Directory used to store dumps"`
MaxDumps int `json:"max-dumps" toml:"max-dumps" comment:"Maximum number of dumps per process"` // maximum number of dump per GUID
Compression bool `json:"compression" toml:"compression" comment:"Enable dumps compression"`
DumpUntracked bool `json:"dump-untracked" toml:"dump-untracked" comment:"Dumps untracked process. Untracked processes are missing\n enrichment information and may generate unwanted dumps"` // whether or not we should dump untracked processes, if true it would create many FPs
Dir string `json:"dir,omitempty" toml:"dir" comment:"Directory used to store dumps"`
MaxDumps int `json:"max-dumps,omitempty" toml:"max-dumps" comment:"Maximum number of dumps per process"` // maximum number of dump per GUID
Compression bool `json:"compression,omitempty" toml:"compression" comment:"Enable dumps compression"`
DumpUntracked bool `json:"dump-untracked,omitempty" toml:"dump-untracked" comment:"Dumps untracked process. Untracked processes are missing\n enrichment information and may generate unwanted dumps"` // whether or not we should dump untracked processes, if true it would create many FPs
}

// Sysmon holds Sysmon related configuration
type Sysmon struct {
Bin string `json:"bin" toml:"bin" comment:"Path to Sysmon binary"`
ArchiveDirectory string `json:"archive-directory" toml:"archive-directory" comment:"Path to Sysmon Archive directory"`
CleanArchived bool `json:"clean-archived" toml:"clean-archived" comment:"Delete files older than 5min archived by Sysmon"`
Bin string `json:"bin,omitempty" toml:"bin" comment:"Path to Sysmon binary"`
ArchiveDirectory string `json:"archive-directory,omitempty" toml:"archive-directory" comment:"Path to Sysmon Archive directory"`
CleanArchived bool `json:"clean-archived,omitempty" toml:"clean-archived" comment:"Delete files older than 5min archived by Sysmon"`
}

// Rules holds rules configuration
type Rules struct {
RulesDB string `json:"rules-db" toml:"rules-db" comment:"Path to Gene rules database"`
ContainersDB string `json:"containers-db" toml:"containers-db" comment:"Path to Gene rules containers\n (c.f. Gene documentation)"`
UpdateInterval time.Duration `json:"update-interval" toml:"update-interval" comment:"Update interval at which rules should be pulled from manager\n NB: only applies if a manager server is configured"`
RulesDB string `json:"rules-db,omitempty" toml:"rules-db" comment:"Path to Gene rules database"`
ContainersDB string `json:"containers-db,omitempty" toml:"containers-db" comment:"Path to Gene rules containers\n (c.f. Gene documentation)"`
UpdateInterval time.Duration `json:"update-interval,omitempty" toml:"update-interval" comment:"Update interval at which rules should be pulled from manager\n NB: only applies if a manager server is configured"`
}

func (c *Rules) RulesPaths() (path, sha256Path string) {
Expand All @@ -62,31 +58,33 @@ func (c *Rules) RulesPaths() (path, sha256Path string) {

// Audit holds Windows audit configuration
type Audit struct {
Enable bool `json:"enable" toml:"enable" comment:"Enable following Audit Policies or not"`
AuditPolicies []string `json:"audit-policies" toml:"audit-policies" comment:"Audit Policies to enable (c.f. auditpol /get /category:* /r)"`
AuditDirs []string `json:"audit-dirs" toml:"audit-dirs" comment:"Set Audit ACL to directories, sub-directories and files to generate File System audit events\n https://docs.microsoft.com/en-us/windows/security/threat-protection/auditing/audit-file-system)"`
Enable bool `json:"enable,omitempty" toml:"enable" comment:"Enable following Audit Policies or not"`
AuditPolicies []string `json:"audit-policies,omitempty" toml:"audit-policies" comment:"Audit Policies to enable (c.f. auditpol /get /category:* /r)"`
AuditDirs []string `json:"audit-dirs,omitempty" toml:"audit-dirs" comment:"Set Audit ACL to directories, sub-directories and files to generate File System audit events\n https://docs.microsoft.com/en-us/windows/security/threat-protection/auditing/audit-file-system)"`
}

// Agent structure
// WARNING: it is very important that any field/structure in Agent config has omitempty in JSON tag otherwise
// there are Sha256 stability issues because JSON and TOML do not decode empty slices the same way.
type Agent struct {
path string

DatabasePath string `json:"db-path" toml:"db-path" comment:"Path to local database root directory"`
CritTresh int `json:"criticality-treshold" toml:"criticality-treshold" comment:"Dumps/forward only events above criticality threshold\n or filtered events (i.e. Gene filtering rules)" `
EnableHooks bool `json:"en-hooks" toml:"en-hooks" comment:"Enable enrichment hooks and dump hooks"`
EnableFiltering bool `json:"en-filters" toml:"en-filters" comment:"Enable event filtering (log filtered events, not only alerts)\n See documentation: https://github.com/0xrawsec/gene" `
Logfile string `json:"logfile" toml:"logfile" comment:"Logfile used to log messages generated by the engine"` // for WHIDS log messages (not alerts)
LogAll bool `json:"log-all" toml:"log-all" comment:"Log any incoming event passing through the engine" ` // log all events to logfile (used for debugging)
Endpoint bool `json:"endpoint" toml:"endpoint" comment:"True if current host is the endpoint on which logs are generated\n Example: turn this off if running on a WEC"`
EtwConfig Etw `json:"etw" toml:"etw" comment:"ETW configuration"`
FwdConfig config.Forwarder `json:"forwarder" toml:"forwarder" comment:"Forwarder configuration"`
Sysmon Sysmon `json:"sysmon" toml:"sysmon" comment:"Sysmon related settings"`
Actions Actions `json:"actions" toml:"actions" comment:"Default actions to apply to events, depending on their criticality"`
Dump Dump `json:"dump" toml:"dump" comment:"Dump related settings"`
Report Report `json:"report" toml:"reporting" comment:"Reporting related settings"`
RulesConfig Rules `json:"rules" toml:"rules" comment:"Gene rules related settings\n Gene repo: https://github.com/0xrawsec/gene\n Gene rules repo: https://github.com/0xrawsec/gene-rules"`
AuditConfig Audit `json:"audit" toml:"audit" comment:"Windows auditing configuration"`
CanariesConfig Canaries `json:"canaries" toml:"canaries" comment:"Canary files configuration"`
path string `json:"path,omitempty"`

DatabasePath string `json:"db-path,omitempty" toml:"db-path" comment:"Path to local database root directory"`
CritTresh int `json:"criticality-treshold,omitempty" toml:"criticality-treshold" comment:"Dumps/forward only events above criticality threshold\n or filtered events (i.e. Gene filtering rules)"`
EnableHooks bool `json:"en-hooks,omitempty" toml:"en-hooks" comment:"Enable enrichment hooks and dump hooks"`
EnableFiltering bool `json:"en-filters,omitempty" toml:"en-filters" comment:"Enable event filtering (log filtered events, not only alerts)\n See documentation: https://github.com/0xrawsec/gene"`
Logfile string `json:"logfile,omitempty" toml:"logfile" comment:"Logfile used to log messages generated by the engine"` // for WHIDS log messages (not alerts)
LogAll bool `json:"log-all,omitempty" toml:"log-all" comment:"Log any incoming event passing through the engine"` // log all events to logfile (used for debugging)
Endpoint bool `json:"endpoint,omitempty" toml:"endpoint" comment:"True if current host is the endpoint on which logs are generated\n Example: turn this off if running on a WEC"`
EtwConfig Etw `json:"etw,omitempty" toml:"etw" comment:"ETW configuration"`
FwdConfig config.Forwarder `json:"forwarder,omitempty" toml:"forwarder" comment:"Forwarder configuration"`
Sysmon Sysmon `json:"sysmon,omitempty" toml:"sysmon" comment:"Sysmon related settings"`
Actions Actions `json:"actions,omitempty" toml:"actions" comment:"Default actions to apply to events, depending on their criticality"`
Dump Dump `json:"dump,omitempty" toml:"dump" comment:"Dump related settings"`
Report Report `json:"report,omitempty" toml:"reporting" comment:"Reporting related settings"`
RulesConfig Rules `json:"rules,omitempty" toml:"rules" comment:"Gene rules related settings\n Gene repo: https://github.com/0xrawsec/gene\n Gene rules repo: https://github.com/0xrawsec/gene-rules"`
AuditConfig Audit `json:"audit,omitempty" toml:"audit" comment:"Windows auditing configuration"`
CanariesConfig Canaries `json:"canaries,omitempty" toml:"canaries" comment:"Canary files configuration"`
}

// LoadAgentConfig loads a HIDS configuration from a file
Expand All @@ -108,30 +106,42 @@ func (c *Agent) Sha256() (string, error) {

// IsForwardingEnabled returns true if a forwarder is actually configured to forward logs
func (c *Agent) IsForwardingEnabled() bool {
return c.FwdConfig != emptyForwarderConfig && !c.FwdConfig.Local
return !c.FwdConfig.Local && c.FwdConfig.Client.HasConnectionSettings()
}

// Prepare creates directory used in the config if not existing
func (c *Agent) Prepare() {
func (c *Agent) Prepare() (err error) {
if !fsutil.Exists(c.RulesConfig.RulesDB) {
os.MkdirAll(c.RulesConfig.RulesDB, 0600)
if err = os.MkdirAll(c.RulesConfig.RulesDB, 0600); err != nil {
return
}
}

if !fsutil.Exists(c.RulesConfig.ContainersDB) {
os.MkdirAll(c.RulesConfig.ContainersDB, 0600)
if err = os.MkdirAll(c.RulesConfig.ContainersDB, 0600); err != nil {
return
}
}

if !fsutil.Exists(c.Dump.Dir) {
os.MkdirAll(c.Dump.Dir, 0600)
if err = os.MkdirAll(c.Dump.Dir, 0600); err != nil {
return
}
}

if !fsutil.Exists(filepath.Dir(c.FwdConfig.Logging.Dir)) {
os.MkdirAll(filepath.Dir(c.FwdConfig.Logging.Dir), 0600)
if err = os.MkdirAll(filepath.Dir(c.FwdConfig.Logging.Dir), 0600); err != nil {
return
}
}

if !fsutil.Exists(filepath.Dir(c.Logfile)) {
os.MkdirAll(filepath.Dir(c.Logfile), 0600)
if err = os.MkdirAll(filepath.Dir(c.Logfile), 0600); err != nil {
return
}
}

return
}

// Verify validate HIDS configuration object
Expand All @@ -153,7 +163,7 @@ func (c *Agent) Path() string {
func (c *Agent) Save(path string) (err error) {
var b []byte

if b, err = utils.Json(c); err != nil {
if b, err = utils.Toml(c); err != nil {
return
}

Expand Down
Loading

0 comments on commit e78e9c8

Please sign in to comment.