Skip to content

Commit

Permalink
Infra for upgrading map from one version to the other. (projectcalico…
Browse files Browse the repository at this point in the history
…#6181)

* Infra for upgrading map from one version to the other.
  • Loading branch information
sridhartigera authored Jun 13, 2022
1 parent 179036c commit 8144afe
Show file tree
Hide file tree
Showing 10 changed files with 975 additions and 4 deletions.
2 changes: 2 additions & 0 deletions felix/.gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@
/artifacts/
/bpf-gpl/include/libbpf/src/amd64/
/bpf-gpl/include/libbpf/src/arm64/
bpf-gpl/include/libbpf/src/armv7/
bpf-gpl/include/libbpf/src/ppc64le/

# IDE files.
/.idea/
Expand Down
50 changes: 50 additions & 0 deletions felix/bpf/bpfmap/upgrade/upgrade.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
// Copyright (c) 2022 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package upgrade

import (
"reflect"

"github.com/projectcalico/calico/felix/bpf"
"github.com/projectcalico/calico/felix/bpf/cachingmap"
)

func UpgradeBPFMap(oldMap, newMap *bpf.PinnedMap) error {
oldVersion := oldMap.Version
newVersion := newMap.Version
oldCachingMap := cachingmap.New(oldMap.MapParameters, oldMap)
newCachingMap := cachingmap.New(newMap.MapParameters, newMap)
err := oldCachingMap.LoadCacheFromDataplane()
if err != nil {
return err
}
err = newCachingMap.LoadCacheFromDataplane()
if err != nil {
return err
}
oldCachingMap.IterDataplaneCache(func(k, v []byte) {
tmpK, tmpV := newMap.KVasUpgradable(oldVersion, k, v)
for i := oldVersion; i < newVersion; i++ {
tmpK = tmpK.Upgrade()
tmpV = tmpV.Upgrade()
}
val := newCachingMap.GetDataplaneCache(tmpK.AsBytes())
if !reflect.DeepEqual(val, tmpV.AsBytes()) {
newCachingMap.SetDesired(tmpK.AsBytes(), tmpV.AsBytes())
}
})
return newCachingMap.ApplyUpdatesOnly()

}
112 changes: 109 additions & 3 deletions felix/bpf/maps.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright (c) 2019-2020 Tigera, Inc. All rights reserved.
// Copyright (c) 2019-2022 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand All @@ -17,9 +17,12 @@ package bpf
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"reflect"
"strconv"
"strings"

"golang.org/x/sys/unix"
Expand Down Expand Up @@ -126,6 +129,10 @@ type PinnedMap struct {
oldfd MapFD
perCPU bool
oldSize int
// Callbacks to handle upgrade
UpgradeFn func(*PinnedMap, *PinnedMap) error
GetMapParams func(int) MapParameters
KVasUpgradable func(int, []byte, []byte) (Upgradable, Upgradable)
}

func (b *PinnedMap) GetName() string {
Expand Down Expand Up @@ -526,8 +533,9 @@ func (b *PinnedMap) EnsureExists() error {
b.fd, err = GetMapFDByPin(b.versionedFilename())
if err == nil {
b.fdLoaded = true
// Copy data from old map to the new map
if copyData {
// Copy data from old map to the new map. Old map and new map are of the
// same version but of different size.
err := b.copyFromOldMap()
if err != nil {
logrus.WithError(err).Error("error copying data from old map")
Expand All @@ -541,6 +549,8 @@ func (b *PinnedMap) EnsureExists() error {
}

}
// Handle map upgrade.
err = b.upgrade()
logrus.WithField("fd", b.fd).WithField("name", b.versionedFilename()).
Info("Loaded map file descriptor.")
}
Expand All @@ -552,6 +562,26 @@ type bpftoolMapMeta struct {
Name string `json:"name"`
}

func GetMapIdFromPin(pinPath string) (int, error) {
cmd := exec.Command("bpftool", "map", "list", "pinned", pinPath, "-j")
out, err := cmd.Output()
if err != nil {
return -1, errors.Wrap(err, "bpftool map list failed")
}

var mapData bpftoolMapMeta
err = json.Unmarshal(out, &mapData)
if err != nil {
return -1, errors.Wrap(err, "bpftool returned bad JSON")
}
return mapData.ID, nil
}

func RepinMapFromId(id int, path string) error {
cmd := exec.Command("bpftool", "map", "pin", "id", fmt.Sprint(id), path)
return errors.Wrap(cmd.Run(), "bpftool failed to reping map from id")
}

func RepinMap(name string, filename string) error {
cmd := exec.Command("bpftool", "map", "list", "-j")
out, err := cmd.Output()
Expand All @@ -578,6 +608,13 @@ func RepinMap(name string, filename string) error {
}

func (b *PinnedMap) CopyDeltaFromOldMap() error {
// check if there is any old version of the map.
// If so upgrade delta entries from the old map
// to the new map.
err := b.upgrade()
if err != nil {
return fmt.Errorf("error upgrading data from old map %s, err=%w", b.GetName(), err)
}
if b.oldfd == 0 {
return nil
}
Expand All @@ -588,9 +625,78 @@ func (b *PinnedMap) CopyDeltaFromOldMap() error {
os.Remove(b.Path() + "_old")
}()

err := b.updateDeltaEntries()
err = b.updateDeltaEntries()
if err != nil {
return fmt.Errorf("error copying data from old map %s, err=%w", b.GetName(), err)
}
return nil
}

func (b *PinnedMap) getOldMapVersion() (int, error) {
oldVersion := 0
dir, name := filepath.Split(b.Filename)
files, err := ioutil.ReadDir(dir)
if err != nil {
return 0, fmt.Errorf("error reading pin path %w", err)
}
for _, f := range files {
fname := f.Name()
fname = string(fname[0 : len(fname)-1])
if fname == name {
mapName := f.Name()
oldVersion, err = strconv.Atoi(string(mapName[len(mapName)-1]))
if err != nil {
return 0, fmt.Errorf("invalid version %w", err)
}
if oldVersion < b.Version {
return oldVersion, nil
}
if oldVersion > b.Version {
return 0, fmt.Errorf("downgrade not supported %d %d", oldVersion, b.Version)
}
oldVersion = 0
}
}
return oldVersion, nil
}

// This function upgrades entries from one version of the map to the other.
// Say we move from mapv2 to mapv3. Data from v2 is upgraded to v3.
// If there is a resized version of v2, which is v2_old, data is upgraded from
// v2_old as well to v3.
func (b *PinnedMap) upgrade() error {
if b.UpgradeFn == nil {
return nil
}
if b.GetMapParams == nil || b.KVasUpgradable == nil {
return fmt.Errorf("upgrade callbacks not registered %s", b.Name)
}
oldVersion, err := b.getOldMapVersion()
if err != nil {
return err
}
// fresh install
if oldVersion == 0 {
return nil
}

// Get a pinnedMap handle for the old map
ctx := b.context
oldMapParams := b.GetMapParams(oldVersion)
oldMapParams.MaxEntries = b.MaxEntries
oldBpfMap := ctx.NewPinnedMap(oldMapParams)
err = oldBpfMap.EnsureExists()
if err != nil {
return err
}
defer func() {
oldBpfMap.(*PinnedMap).Close()
oldBpfMap.(*PinnedMap).fd = 0
}()
return b.UpgradeFn(oldBpfMap.(*PinnedMap), b)
}

type Upgradable interface {
Upgrade() Upgradable
AsBytes() []byte
}
122 changes: 122 additions & 0 deletions felix/bpf/mock/multiversion/map.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
// Copyright (c) 2022 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package multiversion

import (
"github.com/projectcalico/calico/felix/bpf"
"github.com/projectcalico/calico/felix/bpf/bpfmap/upgrade"
v2 "github.com/projectcalico/calico/felix/bpf/mock/multiversion/v2"
v3 "github.com/projectcalico/calico/felix/bpf/mock/multiversion/v3"
v4 "github.com/projectcalico/calico/felix/bpf/mock/multiversion/v4"
v5 "github.com/projectcalico/calico/felix/bpf/mock/multiversion/v5"
)

func GetMapParams(version int) bpf.MapParameters {
switch version {
case 2:
return v2.MockMapParams
case 3:
return v3.MockMapParams
case 4:
return v4.MockMapParams
case 5:
return v5.MockMapParams
default:
return v5.MockMapParams
}
}

func GetKeyValueTypeFromVersion(version int, k, v []byte) (bpf.Upgradable, bpf.Upgradable) {
switch version {
case 2:
var key v2.Key
var val v2.Value
copy(key[:], k)
copy(val[:], v)
return key, val
case 3:
var key v3.Key
var val v3.Value
copy(key[:], k)
copy(val[:], v)
return key, val
case 4:
var key v4.Key
var val v4.Value
copy(key[:], k)
copy(val[:], v)
return key, val
case 5:
var key v5.Key
var val v5.Value
copy(key[:], k)
copy(val[:], v)
return key, val
default:
var key v5.Key
var val v5.Value
copy(key[:], k)
copy(val[:], v)
return key, val
}
}

func MapV2(mc *bpf.MapContext, maxEntries int) bpf.Map {
mockParams := v2.MockMapParams
if maxEntries > 0 {
mockParams.MaxEntries = maxEntries
}
b := mc.NewPinnedMap(mockParams)
b.(*bpf.PinnedMap).UpgradeFn = upgrade.UpgradeBPFMap
b.(*bpf.PinnedMap).GetMapParams = GetMapParams
b.(*bpf.PinnedMap).KVasUpgradable = GetKeyValueTypeFromVersion
return b
}

func MapV3(mc *bpf.MapContext, maxEntries int) bpf.Map {
mockParams := v3.MockMapParams
if maxEntries > 0 {
mockParams.MaxEntries = maxEntries
}
b := mc.NewPinnedMap(mockParams)
b.(*bpf.PinnedMap).UpgradeFn = upgrade.UpgradeBPFMap
b.(*bpf.PinnedMap).GetMapParams = GetMapParams
b.(*bpf.PinnedMap).KVasUpgradable = GetKeyValueTypeFromVersion
return b
}

func MapV4(mc *bpf.MapContext, maxEntries int) bpf.Map {
mockParams := v4.MockMapParams
if maxEntries > 0 {
mockParams.MaxEntries = maxEntries
}
b := mc.NewPinnedMap(mockParams)
b.(*bpf.PinnedMap).UpgradeFn = upgrade.UpgradeBPFMap
b.(*bpf.PinnedMap).GetMapParams = GetMapParams
b.(*bpf.PinnedMap).KVasUpgradable = GetKeyValueTypeFromVersion
return b
}

func MapV5(mc *bpf.MapContext, maxEntries int) bpf.Map {
mockParams := v5.MockMapParams
if maxEntries > 0 {
mockParams.MaxEntries = maxEntries
}
b := mc.NewPinnedMap(mockParams)
b.(*bpf.PinnedMap).UpgradeFn = upgrade.UpgradeBPFMap
b.(*bpf.PinnedMap).GetMapParams = GetMapParams
b.(*bpf.PinnedMap).KVasUpgradable = GetKeyValueTypeFromVersion
return b
}
Loading

0 comments on commit 8144afe

Please sign in to comment.