Skip to content

Commit

Permalink
get private IPs from IPAM and not from instance API
Browse files Browse the repository at this point in the history
  • Loading branch information
@Mia-Cross
Mia-Cross committed Oct 13, 2023
1 parent 542085c commit 85f41b8
Show file tree
Hide file tree
Showing 6 changed files with 164 additions and 73 deletions.
38 changes: 24 additions & 14 deletions protokube/pkg/gossip/scaleway/seeds.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ package scaleway
import (
"fmt"

"github.com/scaleway/scaleway-sdk-go/api/instance/v1"
"github.com/scaleway/scaleway-sdk-go/scw"
"k8s.io/klog/v2"
"k8s.io/kops/protokube/pkg/gossip"
Expand All @@ -43,27 +42,38 @@ func NewSeedProvider(scwClient *scw.Client, clusterName string) (*SeedProvider,
func (p *SeedProvider) GetSeeds() ([]string, error) {
var seeds []string

instanceAPI := instance.NewAPI(p.scwClient)
zone, ok := p.scwClient.GetDefaultZone()
if !ok {
return nil, fmt.Errorf("could not determine default zone from client")
}
klog.V(4).Infof("Found zone of the running server: %v", zone)

region, ok := p.scwClient.GetDefaultRegion()
if !ok {
return nil, fmt.Errorf("could not determine default region from client")
}
servers, err := instanceAPI.ListServers(&instance.ListServersRequest{
Zone: zone,
Tags: []string{fmt.Sprintf("%s=%s", scaleway.TagClusterName, p.tag)},
}, scw.WithAllPages())
klog.V(4).Infof("Found region of the running server: %v", region)

scwCloud, err := scaleway.NewScwCloud(map[string]string{
"region": region.String(),
"zone": zone.String(),
})
if err != nil {
return nil, fmt.Errorf("could not create Scaleway cloud interface: %w", err)
}

servers, err := scwCloud.GetClusterServers(p.tag, nil)
if err != nil {
return nil, fmt.Errorf("failed to get matching servers: %s", err)
return nil, fmt.Errorf("failed to get matching servers: %w", err)
}

for _, server := range servers.Servers {
if server.PrivateIP == nil || *server.PrivateIP == "" {
klog.Warningf("failed to find private ip of the server %s(%s)", server.Name, server.ID)
continue
for _, server := range servers {
ip, err := scwCloud.GetServerIP(server.ID, server.Zone)
if err != nil {
return nil, fmt.Errorf("getting server IP: %w", err)
}

klog.V(4).Infof("Appending gossip seed %s(%s): %q", server.Name, server.ID, *server.PrivateIP)
seeds = append(seeds, *server.PrivateIP)
klog.V(4).Infof("Appending gossip seed %s(%s): %q", server.Name, server.ID, ip)
seeds = append(seeds, ip)
}

klog.V(4).Infof("Get seeds function done now")
Expand Down
56 changes: 40 additions & 16 deletions protokube/pkg/protokube/scaleway_volumes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,13 @@ import (
"net"

"github.com/scaleway/scaleway-sdk-go/api/instance/v1"
ipam "github.com/scaleway/scaleway-sdk-go/api/ipam/v1alpha1"
"github.com/scaleway/scaleway-sdk-go/scw"
"k8s.io/klog/v2"
kopsv "k8s.io/kops"
"k8s.io/kops/protokube/pkg/gossip"
gossipscw "k8s.io/kops/protokube/pkg/gossip/scaleway"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup/scaleway"
)

Expand All @@ -43,28 +45,24 @@ func NewScwCloudProvider() (*ScwCloudProvider, error) {
metadataAPI := instance.NewMetadataAPI()
metadata, err := metadataAPI.GetMetadata()
if err != nil {
return nil, fmt.Errorf("failed to retrieve server metadata: %s", err)
return nil, fmt.Errorf("failed to retrieve server metadata: %w", err)
}

serverID := metadata.ID
klog.V(4).Infof("Found ID of the running server: %v", serverID)

zoneID := metadata.Location.ZoneID
zone, err := scw.ParseZone(zoneID)
zone, err := scw.ParseZone(metadata.Location.ZoneID)
if err != nil {
return nil, fmt.Errorf("unable to parse Scaleway zone: %s", err)
return nil, fmt.Errorf("unable to parse Scaleway zone: %w", err)
}
klog.V(4).Infof("Found zone of the running server: %v", zone)

region, err := scaleway.ParseRegionFromZone(zone)
region, err := zone.Region()
if err != nil {
return nil, fmt.Errorf("unable to parse Scaleway region: %s", err)
return nil, fmt.Errorf("unable to parse Scaleway region: %w", err)
}
klog.V(4).Infof("Found region of the running server: %v", region)

privateIP := metadata.PrivateIP
klog.V(4).Infof("Found first private net IP of the running server: %q", privateIP)

profile, err := scaleway.CreateValidScalewayProfile()
if err != nil {
return nil, err
Expand All @@ -76,23 +74,49 @@ func NewScwCloudProvider() (*ScwCloudProvider, error) {
scw.WithDefaultRegion(region),
)
if err != nil {
return nil, fmt.Errorf("error creating client for Protokube: %w", err)
return nil, fmt.Errorf("creating client for Protokube: %w", err)
}

instanceAPI := instance.NewAPI(scwClient)
server, err := instanceAPI.GetServer(&instance.GetServerRequest{
serverResponse, err := instanceAPI.GetServer(&instance.GetServerRequest{
ServerID: serverID,
Zone: zone,
})
if err != nil || server == nil {
return nil, fmt.Errorf("failed to get the running server: %s", err)
if err != nil || serverResponse.Server == nil {
return nil, fmt.Errorf("failed to get the running server: %w", err)
}
server := serverResponse.Server
klog.V(4).Infof("Found the running server: %q", server.Name)

ips, err := ipam.NewAPI(scwClient).ListIPs(&ipam.ListIPsRequest{
Region: region,
ResourceID: fi.PtrTo(serverID),
IsIPv6: fi.PtrTo(false),
Zonal: fi.PtrTo(zone.String()),
}, scw.WithAllPages())
if err != nil {
return nil, fmt.Errorf("listing server's IPs: %w", err)
}
if ips.TotalCount < 1 {
return nil, fmt.Errorf("expected at least 1 IP attached to the server %s", server.ID)
}

var ipToReturn string
for _, ipFound := range ips.IPs {
if ipFound.Address.IP.IsPrivate() == true {
ipToReturn = ipFound.Address.IP.String()
break
}
}
if ipToReturn == "" {
ipToReturn = ips.IPs[0].Address.IP.String()
}
klog.V(4).Infof("Found the running server: %q", server.Server.Name)
klog.V(4).Infof("Found first private net IP of the running server: %q", ipToReturn)

s := &ScwCloudProvider{
scwClient: scwClient,
server: server.Server,
serverIP: net.IP(privateIP),
server: server,
serverIP: net.IP(ipToReturn),
}

return s, nil
Expand Down
79 changes: 63 additions & 16 deletions upup/pkg/fi/cloudup/scaleway/cloud.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ type ScwCloud interface {
GetClusterServers(clusterName string, instanceGroupName *string) ([]*instance.Server, error)
GetClusterSSHKeys(clusterName string) ([]*iam.SSHKey, error)
GetClusterVolumes(clusterName string) ([]*instance.Volume, error)
GetServerIP(serverID string, zone scw.Zone) (string, error)

DeleteDNSRecord(record *domain.Record, clusterName string) error
DeleteLoadBalancer(loadBalancer *lb.LB) error
Expand Down Expand Up @@ -110,16 +111,11 @@ type scwCloudImplementation struct {
// NewScwCloud returns a Cloud with a Scaleway Client using the env vars SCW_PROFILE or
// SCW_ACCESS_KEY, SCW_SECRET_KEY and SCW_DEFAULT_PROJECT_ID
func NewScwCloud(tags map[string]string) (ScwCloud, error) {
region, err := scw.ParseRegion(tags["region"])
if err != nil {
return nil, err
}
zone, err := scw.ParseZone(tags["zone"])
if err != nil {
return nil, err
}

var scwClient *scw.Client
var region scw.Region
var zone scw.Zone
var err error

if profileName := os.Getenv("SCW_PROFILE"); profileName == "REDACTED" {
// If the profile is REDACTED, we're running integration tests so no need for authentication
scwClient, err = scw.NewClient(scw.WithoutAuth())
Expand All @@ -138,6 +134,19 @@ func NewScwCloud(tags map[string]string) (ScwCloud, error) {
if err != nil {
return nil, fmt.Errorf("creating client for Scaleway Cloud: %w", err)
}
region = scw.Region(fi.ValueOf(profile.DefaultRegion))
zone = scw.Zone(fi.ValueOf(profile.DefaultZone))
}

if tags != nil {
region, err = scw.ParseRegion(tags["region"])
if err != nil {
return nil, err
}
zone, err = scw.ParseZone(tags["zone"])
if err != nil {
return nil, err
}
}

return &scwCloudImplementation{
Expand All @@ -156,7 +165,13 @@ func NewScwCloud(tags map[string]string) (ScwCloud, error) {
}

func (s *scwCloudImplementation) ClusterName(tags []string) string {
return ClusterNameFromTags(tags)
if tags != nil {
return ClusterNameFromTags(tags)
}
if clusterName, ok := s.tags[TagClusterName]; ok {
return clusterName
}
return ""
}

func (s *scwCloudImplementation) DNS() (dnsprovider.Interface, error) {
Expand Down Expand Up @@ -243,6 +258,10 @@ func (s *scwCloudImplementation) DeregisterInstance(i *cloudinstances.CloudInsta
if err != nil {
return fmt.Errorf("deregistering cloud instance %s of group %q: %w", i.ID, i.CloudInstanceGroup.HumanName, err)
}
serverIP, err := s.GetServerIP(server.Server.ID, server.Server.Zone)
if err != nil {
return fmt.Errorf("deregistering cloud instance %s of group %q: %w", i.ID, i.CloudInstanceGroup.HumanName, err)
}

// We remove the instance's IP from load-balancers
lbs, err := s.GetClusterLoadBalancers(s.ClusterName(server.Server.Tags))
Expand All @@ -258,8 +277,8 @@ func (s *scwCloudImplementation) DeregisterInstance(i *cloudinstances.CloudInsta
return fmt.Errorf("deregistering cloud instance %s of group %q: listing load-balancer's back-ends for instance creation: %w", i.ID, i.CloudInstanceGroup.HumanName, err)
}
for _, backEnd := range backEnds.Backends {
for _, serverIP := range backEnd.Pool {
if serverIP == fi.ValueOf(server.Server.PrivateIP) {
for _, ip := range backEnd.Pool {
if ip == serverIP {
_, err := s.lbAPI.RemoveBackendServers(&lb.ZonedAPIRemoveBackendServersRequest{
Zone: s.zone,
BackendID: backEnd.ID,
Expand Down Expand Up @@ -340,7 +359,7 @@ func (s *scwCloudImplementation) GetCloudGroups(cluster *kops.Cluster, instanceg
continue
}

groups[ig.Name], err = buildCloudGroup(ig, serverGroup, nodeMap)
groups[ig.Name], err = buildCloudGroup(s, ig, serverGroup, nodeMap)
if err != nil {
return nil, fmt.Errorf("failed to build cloud group for instance group %q: %w", ig.Name, err)
}
Expand All @@ -364,7 +383,7 @@ func findServerGroups(s *scwCloudImplementation, clusterName string) (map[string
return serverGroups, nil
}

func buildCloudGroup(ig *kops.InstanceGroup, sg []*instance.Server, nodeMap map[string]*v1.Node) (*cloudinstances.CloudInstanceGroup, error) {
func buildCloudGroup(s *scwCloudImplementation, ig *kops.InstanceGroup, sg []*instance.Server, nodeMap map[string]*v1.Node) (*cloudinstances.CloudInstanceGroup, error) {
cloudInstanceGroup := &cloudinstances.CloudInstanceGroup{
HumanName: ig.Name,
InstanceGroup: ig,
Expand All @@ -388,9 +407,11 @@ func buildCloudGroup(ig *kops.InstanceGroup, sg []*instance.Server, nodeMap map[
cloudInstance.State = cloudinstances.State(server.State)
cloudInstance.MachineType = server.CommercialType
cloudInstance.Roles = append(cloudInstance.Roles, InstanceRoleFromTags(server.Tags))
if server.PrivateIP != nil {
cloudInstance.PrivateIP = *server.PrivateIP
ip, err := s.GetServerIP(server.ID, server.Zone)
if err != nil {
return nil, fmt.Errorf("getting server IP: %w", err)
}
cloudInstance.PrivateIP = ip
}

return cloudInstanceGroup, nil
Expand Down Expand Up @@ -474,6 +495,32 @@ func (s *scwCloudImplementation) GetClusterVolumes(clusterName string) ([]*insta
return volumes.Volumes, nil
}

func (s *scwCloudImplementation) GetServerIP(serverID string, zone scw.Zone) (string, error) {
region, err := zone.Region()
if err != nil {
return "", fmt.Errorf("converting zone %s to region: %w", zone, err)
}

ips, err := s.ipamAPI.ListIPs(&ipam.ListIPsRequest{
Region: region,
IsIPv6: fi.PtrTo(false),
ResourceID: &serverID,
Zonal: fi.PtrTo(zone.String()),
}, scw.WithAllPages())
if err != nil {
return "", fmt.Errorf("listing IPs for server %s: %w", serverID, err)
}

if len(ips.IPs) < 1 {
return "", fmt.Errorf("could not find IP for server %s", serverID)
}
if len(ips.IPs) > 1 {
klog.V(10).Infof("Found more than 1 IP for server %s, using %s", serverID, ips.IPs[0].Address.IP.String())
}

return ips.IPs[0].Address.IP.String(), nil
}

func (s *scwCloudImplementation) DeleteDNSRecord(record *domain.Record, clusterName string) error {
domainName := strings.SplitN(clusterName, ".", 2)[1]
recordDeleteRequest := &domain.UpdateDNSZoneRecordsRequest{
Expand Down
34 changes: 24 additions & 10 deletions upup/pkg/fi/cloudup/scaleway/verifier.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,12 @@ import (
"strings"

"github.com/scaleway/scaleway-sdk-go/api/instance/v1"
ipam "github.com/scaleway/scaleway-sdk-go/api/ipam/v1alpha1"
"github.com/scaleway/scaleway-sdk-go/scw"
kopsv "k8s.io/kops"
"k8s.io/kops/pkg/bootstrap"
"k8s.io/kops/pkg/wellknownports"
"k8s.io/kops/upup/pkg/fi"
)

type ScalewayVerifierOptions struct{}
Expand Down Expand Up @@ -71,6 +73,10 @@ func (v scalewayVerifier) VerifyToken(ctx context.Context, rawRequest *http.Requ
if err != nil {
return nil, fmt.Errorf("unable to parse Scaleway zone %q: %w", metadata.Location.ZoneID, err)
}
region, err := zone.Region()
if err != nil {
return nil, fmt.Errorf("unable to determine region from zone %s", zone)
}

profile, err := CreateValidScalewayProfile()
if err != nil {
Expand All @@ -84,25 +90,33 @@ func (v scalewayVerifier) VerifyToken(ctx context.Context, rawRequest *http.Requ
return nil, fmt.Errorf("creating client for Scaleway Verifier: %w", err)
}

instanceAPI := instance.NewAPI(scwClient)
serverResponse, err := instanceAPI.GetServer(&instance.GetServerRequest{
serverResponse, err := instance.NewAPI(scwClient).GetServer(&instance.GetServerRequest{
ServerID: serverID,
Zone: zone,
}, scw.WithContext(ctx))
if err != nil || serverResponse == nil {
if err != nil || serverResponse == nil || serverResponse.Server == nil {
return nil, fmt.Errorf("failed to get server %s: %w", serverID, err)
}
server := serverResponse.Server

ips, err := ipam.NewAPI(scwClient).ListIPs(&ipam.ListIPsRequest{
Region: region,
ResourceID: fi.PtrTo(server.ID),
IsIPv6: fi.PtrTo(false),
Zonal: fi.PtrTo(zone.String()),
}, scw.WithContext(ctx), scw.WithAllPages())
if err != nil {
return nil, fmt.Errorf("failed to get IP for server %q: %w", server.Name, err)
}
if ips.TotalCount == 0 {
return nil, fmt.Errorf("no IP found for server %q: %w", server.Name, err)
}

addresses := []string(nil)
challengeEndPoints := []string(nil)
if server.PrivateIP != nil {
addresses = append(addresses, *server.PrivateIP)
challengeEndPoints = append(challengeEndPoints, net.JoinHostPort(*server.PrivateIP, strconv.Itoa(wellknownports.NodeupChallenge)))
}
if server.IPv6 != nil {
addresses = append(addresses, server.IPv6.Address.String())
challengeEndPoints = append(challengeEndPoints, net.JoinHostPort(server.IPv6.Address.String(), strconv.Itoa(wellknownports.NodeupChallenge)))
for _, ip := range ips.IPs {
addresses = append(addresses, ip.Address.IP.String())
challengeEndPoints = append(challengeEndPoints, net.JoinHostPort(ip.Address.IP.String(), strconv.Itoa(wellknownports.NodeupChallenge)))
}

result := &bootstrap.VerifyResult{
Expand Down
Loading

0 comments on commit 85f41b8

Please sign in to comment.