Skip to content

Commit

Permalink
Add dedicated iptables chains for iptables hook
Browse files Browse the repository at this point in the history
  • Loading branch information
@Devatoria
Devatoria committed Nov 23, 2017
1 parent 10a7dfb commit b9ab562
Show file tree
Hide file tree
Showing 4 changed files with 117 additions and 55 deletions.
2 changes: 1 addition & 1 deletion .travis.yml
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
dist: precise
dist: trusty
sudo: required
language: go
go:
Expand Down
6 changes: 6 additions & 0 deletions config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,13 @@ hooks: # Enabled hooks
proc_path: /mnt/proc # Mount path of host proc folder
registering_retry: 100ms # Time to wait before two registering retry
iptables:
chains:
prerouting: EXECUTOR-PREROUTING
forward: EXECUTOR-FORWARD
postrouting: EXECUTOR-POSTROUTING
container_bridge_interface: docker0 # Brigde interface for container network
ip_forwarding: true
ip_masquerading: true
acl:
chain: MYCHAIN
default_allowed_cidr: # Set of default IP (with CIDR) to allow
Expand Down
44 changes: 31 additions & 13 deletions hook/iptables.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ var IptablesHook = Hook{
}
iptablesHookContainerIPCache.Store(containerID, containerIPs)

return generateIptables(containerIPs, portMappings, driver.Append, true)
return generateIptables(containerIPs, portMappings, driver, driver.Append, true)
},
RunPreStop: func(c container.Containerizer, info *mesos.TaskInfo, containerID string) error {
// Do not execute the hook if we are not on bridged network
Expand Down Expand Up @@ -92,7 +92,7 @@ var IptablesHook = Hook{
)
}

return generateIptables(containerIPs, portMappings, driver.Delete, false)
return generateIptables(containerIPs, portMappings, driver, driver.Delete, false)
},
}

Expand All @@ -101,19 +101,39 @@ var IptablesHook = Hook{
func generateIptables(
containerIPs map[string]net.IP,
portMappings []mesos.ContainerInfo_DockerInfo_PortMapping,
driver *iptables.IPTables,
action func(string, string, ...string) error,
stopOnError bool) error {
var err error

// Get docker interface
containerInterface := viper.GetString("iptables.container_bridge_interface")
if containerInterface == "" {
return fmt.Errorf("could not retrieve container brigde interface")
}

// Check if hook dedicated chains exist
preroutingChain := viper.GetString("iptables.chains.prerouting")
_, err = driver.List("nat", preroutingChain)
if err != nil {
return fmt.Errorf("%s prerouting chain doesn't exist in nat table", preroutingChain)
}

forwardChain := viper.GetString("iptables.chains.forward")
_, err = driver.List("filter", forwardChain)
if err != nil {
return fmt.Errorf("%s forward chain doesn't exist in filter table", forwardChain)
}

postroutingChain := viper.GetString("iptables.chains.postrouting")
_, err = driver.List("nat", postroutingChain)
if err != nil {
return fmt.Errorf("%s postrouting chain doesn't exist in nat table", postroutingChain)
}

ipForward := viper.GetBool("iptables.ip_forwarding")
ipMasquerading := viper.GetBool("iptables.ip_masquerading")

// Init errors
var err error
// Iterate over all container IPs, and for each IP, iterate on container/host binded ports.
// Insert needed iptables for each IP and port.
for _, containerIP := range containerIPs {
Expand All @@ -125,7 +145,7 @@ func generateIptables(
containerInterface,
containerIP.String(),
)
err = action("nat", "POSTROUTING", strings.Split(masqueradeRule, " ")...)
err = action("nat", postroutingChain, strings.Split(masqueradeRule, " ")...)
if err != nil {
if stopOnError {
return err
Expand All @@ -136,8 +156,8 @@ func generateIptables(
}

for _, port := range portMappings {
// Insert rule for translating incoming data on host port to container
if ipMasquerading {
// Insert rule for translating incoming data on host port to container
dnatDestination := []string{containerIP.String(), ":", strconv.Itoa(int(port.GetContainerPort()))}
dnatRule := fmt.Sprintf(
iptableHookDnatRuleTemplate,
Expand All @@ -146,26 +166,24 @@ func generateIptables(
strconv.Itoa(int(port.GetHostPort())),
strings.Join(dnatDestination, ""),
)
err = action("nat", "PREROUTING", strings.Split(dnatRule, " ")...)
err = action("nat", preroutingChain, strings.Split(dnatRule, " ")...)
if err != nil {
if stopOnError {
return err
}

logger.GetInstance().Warn(err.Error())
}
}

// Insert rule for masquerading container -> container network flow
if ipMasquerading {
// Insert rule for masquerading container -> container network flow
selfMasqueradeRule := fmt.Sprintf(
iptableHookSelfMasqueradeRuleTemplate,
containerIP.String(),
port.GetProtocol(),
containerIP.String(),
strconv.Itoa(int(port.GetContainerPort())),
)
err = action("nat", "POSTROUTING", strings.Split(selfMasqueradeRule, " ")...)
err = action("nat", postroutingChain, strings.Split(selfMasqueradeRule, " ")...)
if err != nil {
if stopOnError {
return err
Expand All @@ -185,7 +203,7 @@ func generateIptables(
port.GetProtocol(),
strconv.Itoa(int(port.GetContainerPort())),
)
err = action("filter", "FORWARD", strings.Split(forwardInRule, " ")...)
err = action("filter", forwardChain, strings.Split(forwardInRule, " ")...)
if err != nil {
if stopOnError {
return err
Expand All @@ -203,7 +221,7 @@ func generateIptables(
containerIP.String(),
strconv.Itoa(int(port.GetContainerPort())),
)
err = action("filter", "FORWARD", strings.Split(forwardOutRule, " ")...)
err = action("filter", forwardChain, strings.Split(forwardOutRule, " ")...)
if err != nil {
if stopOnError {
return err
Expand Down
120 changes: 79 additions & 41 deletions hook/iptables_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ type IptablesTestSuite struct {
containerInterface string
taskInfo *mesos.TaskInfo
containerIPs map[string]net.IP
preroutingChain string
forwardChain string
postroutingChain string
}

func (s *IptablesTestSuite) SetupTest() {
Expand All @@ -35,6 +38,9 @@ func (s *IptablesTestSuite) SetupTest() {
s.c = types.NewFakeContainerizer() // Generate fake containerizer
s.hook = IptablesHook
s.containerInterface = "docker0"
s.preroutingChain = "EXECUTOR-PREROUTING"
s.forwardChain = "EXECUTOR-FORWARD"
s.postroutingChain = "EXECUTOR-POSTROUTING"
tcpProtocol := "tcp"
udpProtocol := "udp"
s.taskInfo = &mesos.TaskInfo{
Expand Down Expand Up @@ -81,12 +87,44 @@ func (s *IptablesTestSuite) SetupTest() {
},
)

monkey.Patch(viper.GetString, func(string) string {
return s.containerInterface
var gGetString *monkey.PatchGuard
gGetString = monkey.Patch(viper.GetString, func(k string) string {
gGetString.Unpatch()
defer gGetString.Restore()

switch k {
case "iptables.container_bridge_interface":
return s.containerInterface
case "iptables.chains.prerouting":
return s.preroutingChain
case "iptables.chains.forward":
return s.forwardChain
case "iptables.chains.postrouting":
return s.postroutingChain
default:
return viper.GetString(k)
}
})

var gGetBool *monkey.PatchGuard
gGetBool = monkey.Patch(viper.GetBool, func(k string) bool {
gGetBool.Unpatch()
defer gGetBool.Restore()

switch k {
case "iptables.ip_forwarding", "iptables.ip_masquerading":
return true
default:
return viper.GetBool(k)
}
})

driver, _ := iptables.New() // Get iptables driver
s.iptablesDriver = driver

driver.NewChain("nat", s.preroutingChain)
driver.NewChain("filter", s.forwardChain)
driver.NewChain("nat", s.postroutingChain)
}

func (s *IptablesTestSuite) TearDownTest() {
Expand All @@ -107,17 +145,17 @@ func (s *IptablesTestSuite) TearDownTest() {
// - iptables are correctly injected on at hook execution
func (s *IptablesTestSuite) TestIptablesHookRunPostRun() {
// Store the state of the namespace network
BaseForwardRule, _ := s.iptablesDriver.List("filter", "FORWARD")
BasePostRoutingRules, _ := s.iptablesDriver.List("nat", "POSTROUTING")
BasePreRoutingRules, _ := s.iptablesDriver.List("nat", "PREROUTING")
BaseForwardRule, _ := s.iptablesDriver.List("filter", s.forwardChain)
BasePostRoutingRules, _ := s.iptablesDriver.List("nat", s.postroutingChain)
BasePreRoutingRules, _ := s.iptablesDriver.List("nat", s.preroutingChain)

// Injection should not be executed if the network is not in bridge mode
info := &mesos.TaskInfo{}
assert.Nil(s.T(), s.hook.RunPostRun(s.c, info, ""))

forwardRules, _ := s.iptablesDriver.List("filter", "FORWARD")
postRoutingRules, _ := s.iptablesDriver.List("nat", "POSTROUTING")
preRoutingRules, _ := s.iptablesDriver.List("nat", "PREROUTING")
forwardRules, _ := s.iptablesDriver.List("filter", s.forwardChain)
postRoutingRules, _ := s.iptablesDriver.List("nat", s.postroutingChain)
preRoutingRules, _ := s.iptablesDriver.List("nat", s.preroutingChain)

assert.Equal(
s.T(),
Expand All @@ -140,54 +178,54 @@ func (s *IptablesTestSuite) TestIptablesHookRunPostRun() {
// Now test for each table and chain that rule are correctly inserted,
// next to the previous network states
assert.Nil(s.T(), s.hook.RunPostRun(s.c, s.taskInfo, ""))
forwardRules, _ = s.iptablesDriver.List("filter", "FORWARD")
forwardRules, _ = s.iptablesDriver.List("filter", s.forwardChain)
assert.Subset(
s.T(),
forwardRules,
append(
BaseForwardRule,
[]string{
"-A FORWARD -d 172.0.2.1/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 80 -j ACCEPT",
"-A FORWARD -s 172.0.2.1/32 -i docker0 ! -o docker0 -p tcp -m tcp --sport 80 -j ACCEPT",
"-A FORWARD -d 172.0.2.1/32 ! -i docker0 -o docker0 -p udp -m udp --dport 10000 -j ACCEPT",
"-A FORWARD -s 172.0.2.1/32 -i docker0 ! -o docker0 -p udp -m udp --sport 10000 -j ACCEPT",
"-A FORWARD -d 172.0.3.1/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 80 -j ACCEPT",
"-A FORWARD -s 172.0.3.1/32 -i docker0 ! -o docker0 -p tcp -m tcp --sport 80 -j ACCEPT",
"-A FORWARD -d 172.0.3.1/32 ! -i docker0 -o docker0 -p udp -m udp --dport 10000 -j ACCEPT",
"-A FORWARD -s 172.0.3.1/32 -i docker0 ! -o docker0 -p udp -m udp --sport 10000 -j ACCEPT",
"-A " + s.forwardChain + " -d 172.0.2.1/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 80 -j ACCEPT",
"-A " + s.forwardChain + " -s 172.0.2.1/32 -i docker0 ! -o docker0 -p tcp -m tcp --sport 80 -j ACCEPT",
"-A " + s.forwardChain + " -d 172.0.2.1/32 ! -i docker0 -o docker0 -p udp -m udp --dport 10000 -j ACCEPT",
"-A " + s.forwardChain + " -s 172.0.2.1/32 -i docker0 ! -o docker0 -p udp -m udp --sport 10000 -j ACCEPT",
"-A " + s.forwardChain + " -d 172.0.3.1/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 80 -j ACCEPT",
"-A " + s.forwardChain + " -s 172.0.3.1/32 -i docker0 ! -o docker0 -p tcp -m tcp --sport 80 -j ACCEPT",
"-A " + s.forwardChain + " -d 172.0.3.1/32 ! -i docker0 -o docker0 -p udp -m udp --dport 10000 -j ACCEPT",
"-A " + s.forwardChain + " -s 172.0.3.1/32 -i docker0 ! -o docker0 -p udp -m udp --sport 10000 -j ACCEPT",
}...,
),
forwardRules,
)
postRoutingRules, _ = s.iptablesDriver.List("nat", "POSTROUTING")
postRoutingRules, _ = s.iptablesDriver.List("nat", s.postroutingChain)
assert.Subset(
s.T(),
postRoutingRules,
append(
BasePostRoutingRules,
[]string{
"-A POSTROUTING -s 172.0.2.1/32 ! -o docker0 -j MASQUERADE",
"-A POSTROUTING -s 172.0.2.1/32 -d 172.0.2.1/32 -p tcp -m tcp --dport 80 -j MASQUERADE",
"-A POSTROUTING -s 172.0.2.1/32 -d 172.0.2.1/32 -p udp -m udp --dport 10000 -j MASQUERADE",
"-A POSTROUTING -s 172.0.3.1/32 ! -o docker0 -j MASQUERADE",
"-A POSTROUTING -s 172.0.3.1/32 -d 172.0.3.1/32 -p tcp -m tcp --dport 80 -j MASQUERADE",
"-A POSTROUTING -s 172.0.3.1/32 -d 172.0.3.1/32 -p udp -m udp --dport 10000 -j MASQUERADE",
"-A " + s.postroutingChain + " -s 172.0.2.1/32 ! -o docker0 -j MASQUERADE",
"-A " + s.postroutingChain + " -s 172.0.2.1/32 -d 172.0.2.1/32 -p tcp -m tcp --dport 80 -j MASQUERADE",
"-A " + s.postroutingChain + " -s 172.0.2.1/32 -d 172.0.2.1/32 -p udp -m udp --dport 10000 -j MASQUERADE",
"-A " + s.postroutingChain + " -s 172.0.3.1/32 ! -o docker0 -j MASQUERADE",
"-A " + s.postroutingChain + " -s 172.0.3.1/32 -d 172.0.3.1/32 -p tcp -m tcp --dport 80 -j MASQUERADE",
"-A " + s.postroutingChain + " -s 172.0.3.1/32 -d 172.0.3.1/32 -p udp -m udp --dport 10000 -j MASQUERADE",
}...,
),
postRoutingRules,
)

preRoutingRules, _ = s.iptablesDriver.List("nat", "PREROUTING")
preRoutingRules, _ = s.iptablesDriver.List("nat", s.preroutingChain)
assert.Subset(
s.T(),
preRoutingRules,
append(
BasePreRoutingRules,
[]string{
"-A PREROUTING ! -i docker0 -p tcp -m tcp --dport 32000 -j DNAT --to-destination 172.0.2.1:80",
"-A PREROUTING ! -i docker0 -p udp -m udp --dport 33000 -j DNAT --to-destination 172.0.2.1:10000",
"-A PREROUTING ! -i docker0 -p tcp -m tcp --dport 32000 -j DNAT --to-destination 172.0.3.1:80",
"-A PREROUTING ! -i docker0 -p udp -m udp --dport 33000 -j DNAT --to-destination 172.0.3.1:10000",
"-A " + s.preroutingChain + " ! -i docker0 -p tcp -m tcp --dport 32000 -j DNAT --to-destination 172.0.2.1:80",
"-A " + s.preroutingChain + " ! -i docker0 -p udp -m udp --dport 33000 -j DNAT --to-destination 172.0.2.1:10000",
"-A " + s.preroutingChain + " ! -i docker0 -p tcp -m tcp --dport 32000 -j DNAT --to-destination 172.0.3.1:80",
"-A " + s.preroutingChain + " ! -i docker0 -p udp -m udp --dport 33000 -j DNAT --to-destination 172.0.3.1:10000",
}...,
),
preRoutingRules,
)
}

Expand All @@ -196,9 +234,9 @@ func (s *IptablesTestSuite) TestIptablesHookRunPostRun() {
// - iptables are correctly removed at task hook execution
func (s *IptablesTestSuite) TestIptablesHookRunPreStop() {
// Store the state of the namespace network
referenceForwardRule, _ := s.iptablesDriver.List("filter", "FORWARD")
referencePostRoutingRules, _ := s.iptablesDriver.List("nat", "POSTROUTING")
referencePreRoutingRules, _ := s.iptablesDriver.List("nat", "PREROUTING")
referenceForwardRule, _ := s.iptablesDriver.List("filter", s.forwardChain)
referencePostRoutingRules, _ := s.iptablesDriver.List("nat", s.postroutingChain)
referencePreRoutingRules, _ := s.iptablesDriver.List("nat", s.preroutingChain)

// Removing should not be executed if the network is not in bridge mode
info := &mesos.TaskInfo{}
Expand All @@ -209,26 +247,26 @@ func (s *IptablesTestSuite) TestIptablesHookRunPreStop() {

// Execute remove iptables hook to remove inserted iptables
assert.Nil(s.T(), s.hook.RunPreStop(s.c, s.taskInfo, ""))
forwardRules, _ := s.iptablesDriver.List("filter", "FORWARD")
postRoutingRules, _ := s.iptablesDriver.List("nat", "POSTROUTING")
preRoutingRules, _ := s.iptablesDriver.List("nat", "PREROUTING")
forwardRules, _ := s.iptablesDriver.List("filter", s.forwardChain)
postRoutingRules, _ := s.iptablesDriver.List("nat", s.postroutingChain)
preRoutingRules, _ := s.iptablesDriver.List("nat", s.preroutingChain)

assert.Equal(
s.T(),
referenceForwardRule,
forwardRules,
referenceForwardRule,
)

assert.Equal(
s.T(),
referencePostRoutingRules,
postRoutingRules,
referencePostRoutingRules,
)

assert.Equal(
s.T(),
referencePreRoutingRules,
preRoutingRules,
referencePreRoutingRules,
)
}

Expand Down

0 comments on commit b9ab562

Please sign in to comment.