Skip to content

Commit

Permalink
Merge branch 'master' of https://github.com/mozilla/mozdef
Browse files Browse the repository at this point in the history
  • Loading branch information
mpurzynski committed Apr 3, 2019
2 parents 0add24b + fb898a2 commit ba33a0c
Show file tree
Hide file tree
Showing 36 changed files with 501 additions and 112 deletions.
42 changes: 39 additions & 3 deletions CHANGELOG
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,46 @@ All notable changes to this project will be documented in this file.

The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)

## [Unreleased]

## [v1.38.3] - 2019-04-01
### Fixed
- AWS CodeBuild tag semver regex

## [v1.38.2] - 2019-03-29
### Fixed
- Remaining references to old alertplugins container

## [v1.38.1] - 2019-03-29
### Added
- Enable CI/CD with AWS CodeBuild
- Create AMIs of MozDef, replicate and share them
- Link everything (container images, AMIs, templates) together by MozDef version

### Changed
- Publish versioned CloudFormation templates
- RabbitMQ configured to use a real password

## [v1.38] - 2019-03-28
### Added
- Create alert plugins with ability to modify alerts in pipeline

### Changed
- Renamed existing alertplugin service to alertactions
- Updated rabbitmq docker container to 3.7

### Fixed
- Resolved sshd mq plugin to handle more types of events


## [v1.37] - 2019-03-01
### Added
- Watchlist - use the UI to quickly add a term (username, IP, command, etc.) that MozDef alerts on
- Generic Deadman - use a simple config file to validate that expected events are appearing in a given time window (and alert an Error when they do not)

### Changed
- Improve error handling on Slack bot
- Improve Slack bot alert format for better readibility
- Improve Slack bot alert format for better readability
- Minor UI adjustments

### Fixed
Expand All @@ -19,5 +51,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
- Added checks on sending SQS messages to only accept intra-account messages
- Improved docker performance and disk space requirements

[Unreleased Changes]: https://github.com/mozilla/MozDef/compare/v1.37...HEAD
[Releases prior to v1.37](https://github.com/mozilla/MozDef/releases)
[Unreleased]: https://github.com/mozilla/MozDef/compare/v1.38.3...HEAD
[v1.38.3]: https://github.com/mozilla/MozDef/compare/v1.38.2...v1.38.3
[v1.38.2]: https://github.com/mozilla/MozDef/compare/v1.38.1...v1.38.2
[v1.38.1]: https://github.com/mozilla/MozDef/compare/v1.38...v1.38.1
[v1.38]: https://github.com/mozilla/MozDef/compare/v1.37...v1.38
[v1.37]: https://github.com/mozilla/MozDef/releases/tag/v1.37
15 changes: 15 additions & 0 deletions CODE_OF_CONDUCT.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# Community Participation Guidelines

This repository is governed by Mozilla's code of conduct and etiquette guidelines.
For more details, please read the
[Mozilla Community Participation Guidelines](https://www.mozilla.org/about/governance/policies/participation/).

## How to Report
For more information on how to report violations of the Community Participation Guidelines, please read our '[How to Report](https://www.mozilla.org/about/governance/policies/participation/reporting/)' page.

<!--
## Project Specific Etiquette
In some cases, there will be additional project etiquette i.e.: (https://bugzilla.mozilla.org/page.cgi?id=etiquette.html).
Please update for your project.
-->
71 changes: 62 additions & 9 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,11 @@ DKR_IMAGES := mozdef_alertactions mozdef_alerts mozdef_base mozdef_bootstrap moz
BUILD_MODE := build ## Pass `pull` in order to pull images instead of building them
NAME := mozdef
VERSION := 0.1
BRANCH := master
NO_CACHE := ## Pass `--no-cache` in order to disable Docker cache
GITHASH := latest ## Pass `$(git rev-parse --short HEAD`) to tag docker hub images as latest git-hash instead
TEST_CASE := tests ## Run all (`tests`) or a specific test case (ex `tests/alerts/tests/alerts/test_proxy_drop_exfil_domains.py`)
TMPDIR := $(shell mktemp -d )

.PHONY:all
all:
Expand All @@ -24,11 +26,11 @@ all:
run: build ## Run all MozDef containers
docker-compose -f docker/compose/docker-compose.yml -p $(NAME) up -d

.PHONY: run-cloudy-mozdef restart-cloudy-mozdef
.PHONY: run-cloudy-mozdef
run-cloudy-mozdef: ## Run the MozDef containers necessary to run in AWS (`cloudy-mozdef`). This is used by the CloudFormation-initiated setup.
$(shell test -f docker/compose/cloudy_mozdef.env || touch docker/compose/cloudy_mozdef.env)
$(shell test -f docker/compose/cloudy_mozdef_kibana.env || touch docker/compose/cloudy_mozdef_kibana.env)
docker-compose -f docker/compose/docker-compose-cloudy-mozdef.yml -p $(NAME) pull
# docker-compose -f docker/compose/docker-compose-cloudy-mozdef.yml -p $(NAME) pull # Images are now in the local packer build AMI and no docker pull is needed
docker-compose -f docker/compose/docker-compose-cloudy-mozdef.yml -p $(NAME) up -d

.PHONY: run-env-mozdef
Expand All @@ -39,58 +41,109 @@ else
@echo $(ENV) not found.
endif

.PHONY: restart-cloudy-mozdef
restart-cloudy-mozdef:
docker-compose -f docker/compose/docker-compose-cloudy-mozdef.yml -p $(NAME) restart

.PHONY: tests run-tests-resources run-tests-resources-external run-tests
.PHONY: test
test: build-tests run-tests

.PHONY: tests
tests: build-tests run-tests ## Run all tests (getting/building images as needed)

.PHONY: run-tests-resources-external
run-tests-resources-external: ## Just spin up external resources for tests and have them listen externally
docker-compose -f docker/compose/docker-compose-tests.yml -p test-$(NAME) run -p 9200:9200 -d elasticsearch
docker-compose -f docker/compose/docker-compose-tests.yml -p test-$(NAME) run -p 5672:5672 -d rabbitmq

.PHONY: run-tests-resources
run-tests-resources: ## Just run the external resources required for tests
docker-compose -f docker/compose/docker-compose-tests.yml -p test-$(NAME) up -d
run-test:

.PHONY: run-test
run-test: run-tests

.PHONY: run-test
run-tests: run-tests-resources ## Just run the tests (no build/get). Use `make TEST_CASE=tests/...` for specific tests only
docker run -it --rm mozdef/mozdef_tester bash -c "source /opt/mozdef/envs/python/bin/activate && flake8 --config .flake8 ./"
docker run -it --rm --network=test-mozdef_default mozdef/mozdef_tester bash -c "source /opt/mozdef/envs/python/bin/activate && py.test --delete_indexes --delete_queues $(TEST_CASE)"
rebuild-run-tests: build-tests run-tests

.PHONY: rebuild-run-tests
rebuild-run-tests: build-tests run-tests

.PHONY: build
build: ## Build local MozDef images (use make NO_CACHE=--no-cache build to disable caching)
build: build-from-cwd

.PHONY: build-from-cwd
build-from-cwd: ## Build local MozDef images (use make NO_CACHE=--no-cache build to disable caching)
docker-compose -f docker/compose/docker-compose.yml -p $(NAME) $(NO_CACHE) $(BUILD_MODE)

.PHONY: build-from-github
build-from-github: ## Build local MozDef images from the github branch (use make NO_CACHE=--no-cache build to disable caching).
@echo "Performing a build from the github branch using $(TMPDIR) for BRANCH=$(BRANCH)"
cd $(TMPDIR) && git clone https://github.com/mozilla/MozDef.git && cd MozDef && git checkout $(BRANCH) && make build-from-cwd
rm -rf $(TMPDIR)

.PHONY: build-tests
build-tests: ## Build end-to-end test environment only
docker-compose -f docker/compose/docker-compose-tests.yml -p test-$(NAME) $(NO_CACHE) $(BUILD_MODE)

.PHONY: stop down
.PHONY: stop
stop: down

.PHONY: down
down: ## Shutdown all services we started with docker-compose
docker-compose -f docker/compose/docker-compose.yml -p $(NAME) stop
docker-compose -f docker/compose/docker-compose.yml -p test-$(NAME) stop

.PHONY: docker-push docker-get hub hub-get
.PHONY: docker-push
docker-push: hub

.PHONY: hub
hub: ## Upload locally built MozDef images tagged as the current git head (hub.docker.com/mozdef).
docker login
docker-compose -f docker/compose/docker-compose.yml -p $(NAME) push
docker-compose -f docker/compose/docker-compose-tests.yml -p test-$(NAME) push

.PHONY: tag-images
tag-images:
cloudy_mozdef/ci/docker_tag_or_push tag $(BRANCH)

.PHONY: docker-push-tagged
docker-push-tagged: tag-images hub-tagged

.PHONY: hub-tagged
hub-tagged: ## Upload locally built MozDef images tagged as the BRANCH. Branch and tagged release are interchangeable here.
cloudy_mozdef/ci/docker_tag_or_push push $(BRANCH)

.PHONY: docker-get
docker-get: hub-get

.PHONY: hub-get
hub-get: ## Download all pre-built images (hub.docker.com/mozdef)
docker-compose -f docker/compose/docker-compose.yml -p $(NAME) pull
docker-compose -f docker/compose/docker-compose-test.yml -p test-$(NAME) pull

.PHONY: docker-login
docker-login: hub-login

.PHONY: hub-login
hub-login: ## Login as the MozDef CI user in order to perform a release of the containers.
@docker login -u mozdefci --password $(shell aws ssm get-parameter --name '/mozdef/ci/dockerhubpassword' --with-decrypt | jq .Parameter.Value)

.PHONY: clean
clean: ## Cleanup all docker volumes and shutdown all related services
-docker-compose -f docker/compose/docker-compose.yml -p $(NAME) down -v --remove-orphans
-docker-compose -f docker/compose/docker-compose-tests.yml -p test-$(NAME) down -v --remove-orphans

# Shorthands
.PHONY: rebuild
rebuild: clean build
rebuild: clean build-from-cwd

.PHONY: new-alert
new-alert: ## Create an example alert and working alert unit test
python tests/alert_templater.py

.PHONY: set-version-and-fetch-docker-container
set-version-and-fetch-docker-container: build-from-cwd tag-images # Lock the release of MozDef by pulling the docker containers on AMI build and caching replace all instances of latest in the compose override with the BRANCH
sed -i s/latest/$(BRANCH)/g docker/compose/docker-compose-cloudy-mozdef.yml
Empty file added alerts/plugins/__init__.py
Empty file.
30 changes: 20 additions & 10 deletions cloudy_mozdef/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ ROOT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
PARENTDIR := $(realpath ../)
AWS_REGION := us-west-2
STACK_NAME := mozdef-aws-nested
BRANCH := master
AMI_MAP_TEMP_FILE := /tmp/mozdef-ami-map.txt
DEV_STACK_PARAMS_FILENAME := aws_parameters.dev.json
# For more information on the rationale behind the code in STACK_PARAMS see https://github.com/aws/aws-cli/issues/2429#issuecomment-441133480
DEV_STACK_PARAMS := $(shell test -e $(DEV_STACK_PARAMS_FILENAME) && python -c 'import json,sys;f=open(sys.argv[1]);print(" ".join([",".join(["%s=\\\"%s\\\""%(k,v) for k,v in x.items()]) for x in json.load(f)]));f.close()' $(DEV_STACK_PARAMS_FILENAME))
Expand All @@ -20,24 +22,26 @@ S3_PROD_STACK_URI := https://s3-$(AWS_REGION).amazonaws.com/$(S3_PROD_BUCKET_NAM
# OIDC_CLIENT_SECRET is set in an environment variable by running "source aws_parameters.sh"
OIDC_CLIENT_SECRET_PARAM_ARG := $(shell test -n "$(OIDC_CLIENT_SECRET)" && echo "ParameterKey=OIDCClientSecret,ParameterValue=$(OIDC_CLIENT_SECRET)")

.PHONY:all
all:
@echo 'Available make targets:'
@grep '^[^#[:space:]\.PHONY.*].*:' Makefile
@echo 'Run ./dmake <target> in order to run the Makefile targets in Docker'

# Note: This requires AWS access
.PHONY: packer-build
packer-build: ## Build the base AMI with packer
cd packer && packer build packer.json
# https://blog.gruntwork.io/locating-aws-ami-owner-id-and-image-name-for-packer-builds-7616fe46b49a
.PHONY: packer-build-github
packer-build-github: ## Build the base AMI with packer
@echo "Branch based build triggered for $(BRANCH)."
ci/pack_and_copy $(BRANCH) $(AMI_MAP_TEMP_FILE)

.PHONY: create-prod-stack
.PHONY: create-dev-stack
create-dev-stack: test ## Create everything you need for a fresh new stack!
@export AWS_REGION=$(AWS_REGION)
@echo "Make sure you have an environment variable OIDC_CLIENT_SECRET set."
aws cloudformation create-stack --stack-name $(STACK_NAME) --template-url $(S3_DEV_STACK_URI)mozdef-parent.yml \
--capabilities CAPABILITY_IAM \
--parameters ParameterKey=S3TemplateLocation,ParameterValue=$(S3_DEV_STACK_URI) \
$(OIDC_CLIENT_SECRET_PARAM_ARG) \
--parameters $(OIDC_CLIENT_SECRET_PARAM_ARG) \
$(DEV_STACK_PARAMS) \
--output text

Expand All @@ -46,18 +50,19 @@ create-dev-s3-bucket:
@export AWS_REGION=$(AWS_REGION)
aws s3api create-bucket --bucket $(S3_DEV_BUCKET_NAME) --acl public-read --create-bucket-configuration LocationConstraint=$(AWS_REGION)

.PHONY: updated-dev-stack
.PHONY: update-dev-stack
update-dev-stack: test ## Updates the nested stack on AWS
@export AWS_REGION=$(AWS_REGION)
aws cloudformation update-stack --stack-name $(STACK_NAME) --template-url $(S3_DEV_STACK_URI)mozdef-parent.yml \
--capabilities CAPABILITY_IAM \
--parameters ParameterKey=S3TemplateLocation,ParameterValue=$(S3_DEV_STACK_URI) \
$(OIDC_CLIENT_SECRET_PARAM_ARG) \
--parameters $(OIDC_CLIENT_SECRET_PARAM_ARG) \
$(DEV_STACK_PARAMS) \
--output text

.PHONY: cfn-lint test
.PHONY: test
test: cfn-lint

.PHONY: cfn-lint
cfn-lint: ## Verify the CloudFormation template pass linting tests
-cfn-lint cloudformation/*.yml

Expand All @@ -76,6 +81,11 @@ publish-prod-templates:
@export AWS_REGION=$(AWS_REGION)
aws s3 sync cloudformation/ $(S3_PROD_BUCKET_URI) --exclude="*" --include="*.yml"

.PHONY: publish-versioned-templates
publish-versioned-templates:
@export AWS_REGION=$(AWS_REGION)
ci/publish_versioned_templates $(BRANCH) $(S3_PROD_BUCKET_URI) $(S3_PROD_STACK_URI) $(AMI_MAP_TEMP_FILE)

.PHONY: diff-dev-templates
diff-dev-templates:
tempdir=`mktemp --directory`; aws s3 sync $(S3_DEV_BUCKET_URI) "$$tempdir" --exclude="*" --include="*.yml"; diff --recursive --unified "$$tempdir" cloudformation; rm -rf "$$tempdir"
Expand Down
9 changes: 2 additions & 7 deletions cloudy_mozdef/aws_parameters.example.json
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@
"UsePreviousValue": false
},
{
"ParameterKey": "AMIImageId",
"ParameterValue": "ami-0c3705bb3b43ad51f",
"ParameterKey": "SSHIngressCIDR",
"ParameterValue": "0.0.0.0/0",
"UsePreviousValue": false
},
{
Expand All @@ -43,10 +43,5 @@
"ParameterKey": "OIDCClientSecret",
"ParameterValue": "secret-value-goes-here",
"UsePreviousValue": false
},
{
"ParameterKey": "S3TemplateLocation",
"ParameterValue": "https://s3-us-west-2.amazonaws.com/example-bucket-name/cloudformation/path/",
"UsePreviousValue": false
}
]
14 changes: 14 additions & 0 deletions cloudy_mozdef/buildspec.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
version: 0.2

phases:
install:
commands:
- echo 'Codebuild is ubuntu 14.04 installing packer to compensate Someone should build a ci docker container \;).'
- wget -nv https://releases.hashicorp.com/packer/1.3.5/packer_1.3.5_linux_amd64.zip
- unzip packer_1.3.5_linux_amd64.zip
- chmod +x packer
- mv packer /usr/bin/
build:
commands:
- mkdir -p serverless-functions/build/python/lib/python3.6/site-packages
- bash cloudy_mozdef/ci/deploy
34 changes: 34 additions & 0 deletions cloudy_mozdef/ci/deploy
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
#!/bin/bash

set -e # Exit immediately if a command exits with a non-zero status.

echo 'Welcome GitHub webhook to the CodeBuild Job of MozDef.'
echo "It's dangerous to go alone. Take one of these: <%%%%|==========>"

# echo "Begin test of the MozDef codebase."
# export COMPOSE_INTERACTIVE_NO_CLI=1 make tests
# The above does not currently work in a non-interactive TTY.
# Fails with error
# docker run -it --rm mozdef/mozdef_tester bash -c "source /opt/mozdef/envs/python/bin/activate && flake8 --config .flake8 ./"
# the input device is not a TTY
# make: *** [run-tests] Error 1
# Then again we probably do not need to run the test suite here because it has been run three times to get the code here.
# echo "Tests complete.

echo "Processing webhook event for ${CODEBUILD_WEBHOOK_TRIGGER}."

if [[ "branch/master" == "$CODEBUILD_WEBHOOK_TRIGGER" \
|| "$CODEBUILD_WEBHOOK_TRIGGER" =~ ^tag\/v[0-9]+\.[0-9]+\.[0-9]+(\-(prod|pre|testing))?$ ]]; then
echo "Building a release"
echo "C|_| This may take a bit. Might as well grab a coffee."
make build-from-cwd
cd cloudy_mozdef
BRANCH="`echo $CODEBUILD_WEBHOOK_TRIGGER | cut -d '/' -f2`"
make BRANCH=${BRANCH} packer-build-github
make BRANCH=${BRANCH} publish-versioned-templates
cd ..
make hub-login
make BRANCH=${BRANCH} docker-push-tagged
fi

echo "End build of the MozDef codebase."
22 changes: 22 additions & 0 deletions cloudy_mozdef/ci/docker_tag_or_push
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
#!/bin/bash

action="${1}"
branch="${2}"

for name in mozdef_meteor mozdef_base mozdef_tester mozdef_mq_worker mozdef_kibana \
mozdef_syslog mozdef_cron mozdef_elasticsearch mozdef_loginput mozdef_mongodb \
mozdef_bootstrap mozdef_alerts mozdef_nginx mozdef_alertactions mozdef_rabbitmq \
mozdef_rest mozdef_base ; do
if [ "${action}" == "tag" ]; then
if [ "${branch}" == "master" ]; then
docker tag mozdef/${name}:latest mozdef/${name}:${branch}
else
docker tag mozdef/${name}:${branch}
fi
elif [ "${action}" == "push" ]; then
docker push mozdef/${name}:${branch}
if [ "${branch}" == "master" ]; then
docker push mozdef/${name}:latest
fi
fi
done
Loading

0 comments on commit ba33a0c

Please sign in to comment.