multi path process

BUILD.md

@@ -1,5 +1,6 @@
 # Development Guide
 
+
 ## Building the development environment
 
 You need a working [Go environment](https://golang.org/doc/install) (1.16 or newer).

go.mod

@@ -31,6 +31,7 @@ require (
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/eapache/queue v1.1.0 // indirect
+	github.com/fatih/set v0.2.1 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect

go.sum

@@ -71,6 +71,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
 github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fatih/set v0.2.1 h1:nn2CaJyknWE/6txyUDGwysr3G5QC6xWB/PtVjPBbeaA=
+github.com/fatih/set v0.2.1/go.mod h1:+RKtMCH+favT2+3YecHGxcc0b4KyVWA1QWWJUs4E0CI=
 github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=

internal/pkg/table/destination.go

@@ -521,7 +521,7 @@ func (u *Update) GetWithdrawnPath() []*Path {
 	return l
 }
 
-func (u *Update) GetChanges(id string, as uint32, peerDown bool) (*Path, *Path, []*Path) {
+func (u *Update) GetChanges(id string, as uint32, peerDown bool) (*Path, *Path, []*Path, []*Path) {
 	best, old := func(id string) (*Path, *Path) {
 		old := getBestPath(id, as, u.OldKnownPathList)
 		best := getBestPath(id, as, u.KnownPathList)
@@ -563,6 +563,7 @@ func (u *Update) GetChanges(id string, as uint32, peerDown bool) (*Path, *Path,
 	}(id)
 
 	var multi []*Path
+	var oldMulti []*Path
 
 	if id == GLOBAL_RIB_NAME && UseMultiplePaths.Enabled {
 		diff := func(lhs, rhs []*Path) bool {
@@ -580,12 +581,13 @@ func (u *Update) GetChanges(id string, as uint32, peerDown bool) (*Path, *Path,
 		newM := getMultiBestPath(id, u.KnownPathList)
 		if diff(oldM, newM) {
 			multi = newM
+			oldMulti = oldM
 			if len(newM) == 0 {
 				multi = []*Path{best}
 			}
 		}
 	}
-	return best, old, multi
+	return best, old, multi, oldMulti
 }
 
 func compareByLLGRStaleCommunity(path1, path2 *Path) *Path {

internal/pkg/table/destination_test.go

@@ -349,15 +349,15 @@ func TestMultipath(t *testing.T) {
 	d := NewDestination(nlri[0], 0)
 	d.Calculate(logger, path2)
 
-	best, old, multi := d.Calculate(logger, path1).GetChanges(GLOBAL_RIB_NAME, 0, false)
+	best, old, multi, _ := d.Calculate(logger, path1).GetChanges(GLOBAL_RIB_NAME, 0, false)
 	assert.NotNil(t, best)
 	assert.Equal(t, old, path2)
 	assert.Equal(t, len(multi), 2)
 	assert.Equal(t, len(d.GetKnownPathList(GLOBAL_RIB_NAME, 0)), 2)
 
 	path3 := path2.Clone(true)
 	dd := d.Calculate(logger, path3)
-	best, old, multi = dd.GetChanges(GLOBAL_RIB_NAME, 0, false)
+	best, old, multi, _ = dd.GetChanges(GLOBAL_RIB_NAME, 0, false)
 	assert.Nil(t, best)
 	assert.Equal(t, old, path1)
 	assert.Equal(t, len(multi), 1)
@@ -375,7 +375,7 @@ func TestMultipath(t *testing.T) {
 	updateMsg = bgp.NewBGPUpdateMessage(nil, pathAttributes, nlri)
 	path4 := ProcessMessage(updateMsg, peer3, time.Now())[0]
 	dd = d.Calculate(logger, path4)
-	best, _, multi = dd.GetChanges(GLOBAL_RIB_NAME, 0, false)
+	best, _, multi, _ = dd.GetChanges(GLOBAL_RIB_NAME, 0, false)
 	assert.NotNil(t, best)
 	assert.Equal(t, len(multi), 1)
 	assert.Equal(t, len(d.GetKnownPathList(GLOBAL_RIB_NAME, 0)), 2)
@@ -389,7 +389,7 @@ func TestMultipath(t *testing.T) {
 	}
 	updateMsg = bgp.NewBGPUpdateMessage(nil, pathAttributes, nlri)
 	path5 := ProcessMessage(updateMsg, peer2, time.Now())[0]
-	best, _, multi = d.Calculate(logger, path5).GetChanges(GLOBAL_RIB_NAME, 0, false)
+	best, _, multi, _ = d.Calculate(logger, path5).GetChanges(GLOBAL_RIB_NAME, 0, false)
 	assert.NotNil(t, best)
 	assert.Equal(t, len(multi), 2)
 	assert.Equal(t, len(d.GetKnownPathList(GLOBAL_RIB_NAME, 0)), 3)

internal/pkg/table/table_manager_test.go

@@ -38,7 +38,7 @@ func (manager *TableManager) ProcessUpdate(fromPeer *PeerInfo, message *bgp.BGPM
 		dsts = append(dsts, manager.Update(path)...)
 	}
 	for _, d := range dsts {
-		b, _, _ := d.GetChanges(GLOBAL_RIB_NAME, 0, false)
+		b, _, _, _ := d.GetChanges(GLOBAL_RIB_NAME, 0, false)
 		pathList = append(pathList, b)
 	}
 	return pathList, nil

pkg/server/bmp.go

@@ -23,6 +23,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/fatih/set"
 	api "github.com/osrg/gobgp/v3/api"
 	"github.com/osrg/gobgp/v3/internal/pkg/table"
 	"github.com/osrg/gobgp/v3/pkg/config/oc"
@@ -213,12 +214,66 @@ func (b *bmpClient) loop() {
 							AS:      b.s.bgpConfig.Global.Config.As,
 							ID:      net.ParseIP(b.s.bgpConfig.Global.Config.RouterId).To4(),
 						}
-						for _, p := range msg.PathList {
-							u := table.CreateUpdateMsgFromPaths([]*table.Path{p})[0]
-							if payload, err := u.Serialize(); err != nil {
-								return false
-							} else if err = write(bmpPeerRoute(bmp.BMP_PEER_TYPE_LOCAL_RIB, false, 0, true, info, p.GetTimestamp().Unix(), payload)); err != nil {
-								return false
+						if table.UseMultiplePaths.Enabled {
+							var hMap = make(map[string]int)
+							var newSet = set.New(set.NonThreadSafe)
+							var oldSet = set.New(set.NonThreadSafe)
+
+							if len(msg.MultiPathList) != 0 {
+								for i, p := range msg.MultiPathList[0] {
+									if p.IsWithdraw {
+										continue
+									}
+
+									nh := p.GetNexthop().String()
+									newSet.Add(nh)
+									hMap[nh] = i
+								}
+							}
+
+							if len(msg.OldMultiPathList) != 0 {
+								for i, p := range msg.OldMultiPathList[0] {
+									nh := p.GetNexthop().String()
+									oldSet.Add(nh)
+									hMap[nh] = i
+								}
+							}
+
+							delSet := set.Difference(oldSet, newSet)
+							addSet := set.Difference(newSet, oldSet)
+
+							for _, i := range delSet.List() {
+								nh := i.(string)
+								p := msg.OldMultiPathList[0][hMap[nh]]
+								p.IsWithdraw = true
+
+								u := table.CreateUpdateMsgFromPaths([]*table.Path{p})[0]
+								if payload, err := u.Serialize(); err != nil {
+									return false
+								} else if err = write(bmpPeerRoute(bmp.BMP_PEER_TYPE_LOCAL_RIB, false, 0, true, info, p.GetTimestamp().Unix(), payload)); err != nil {
+									return false
+								}
+							}
+
+							for _, i := range addSet.List() {
+								nh := i.(string)
+								p := msg.MultiPathList[0][hMap[nh]]
+
+								u := table.CreateUpdateMsgFromPaths([]*table.Path{p})[0]
+								if payload, err := u.Serialize(); err != nil {
+									return false
+								} else if err = write(bmpPeerRoute(bmp.BMP_PEER_TYPE_LOCAL_RIB, false, 0, true, info, p.GetTimestamp().Unix(), payload)); err != nil {
+									return false
+								}
+							}
+						} else {
+							for _, p := range msg.PathList {
+								u := table.CreateUpdateMsgFromPaths([]*table.Path{p})[0]
+								if payload, err := u.Serialize(); err != nil {
+									return false
+								} else if err = write(bmpPeerRoute(bmp.BMP_PEER_TYPE_LOCAL_RIB, false, 0, true, info, p.GetTimestamp().Unix(), payload)); err != nil {
+									return false
+								}
 							}
 						}
 					case *watchEventPeer:

pkg/server/server.go

@@ -800,7 +800,7 @@ func (s *BgpServer) setPathVrfIdMap(paths []*table.Path, m map[uint32]bool) {
 
 // Note: the destination would be the same for all the paths passed here
 // The wather (only zapi) needs a unique list of vrf IDs
-func (s *BgpServer) notifyBestWatcher(best []*table.Path, multipath [][]*table.Path) {
+func (s *BgpServer) notifyBestWatcher(best []*table.Path, multipath [][]*table.Path, oldmultipath [][]*table.Path) {
 	if table.SelectionOptions.DisableBestPathSelection {
 		// Note: If best path selection disabled, no best path to notify.
 		return
@@ -813,11 +813,18 @@ func (s *BgpServer) notifyBestWatcher(best []*table.Path, multipath [][]*table.P
 			s.setPathVrfIdMap(clonedM[i], m)
 		}
 	}
+	oldClonedM := make([][]*table.Path, len(oldmultipath))
+	for i, pathList := range oldmultipath {
+		oldClonedM[i] = clonePathList(pathList)
+		if table.UseMultiplePaths.Enabled {
+			s.setPathVrfIdMap(oldClonedM[i], m)
+		}
+	}
 	clonedB := clonePathList(best)
 	if !table.UseMultiplePaths.Enabled {
 		s.setPathVrfIdMap(clonedB, m)
 	}
-	w := &watchEventBestPath{PathList: clonedB, MultiPathList: clonedM}
+	w := &watchEventBestPath{PathList: clonedB, MultiPathList: clonedM, OldMultiPathList: oldClonedM}
 	if len(m) > 0 {
 		w.Vrf = m
 	}
@@ -1281,20 +1288,25 @@ func (s *BgpServer) propagateUpdate(peer *peer, pathList []*table.Path) {
 	}
 }
 
-func dstsToPaths(id string, as uint32, dsts []*table.Update) ([]*table.Path, []*table.Path, [][]*table.Path) {
+func dstsToPaths(id string, as uint32, dsts []*table.Update) ([]*table.Path, []*table.Path, [][]*table.Path, [][]*table.Path) {
 	bestList := make([]*table.Path, 0, len(dsts))
 	oldList := make([]*table.Path, 0, len(dsts))
 	mpathList := make([][]*table.Path, 0, len(dsts))
+	oldmpathList := make([][]*table.Path, 0, len(dsts))
 
 	for _, dst := range dsts {
-		best, old, mpath := dst.GetChanges(id, as, false)
+		best, old, mpath, oldmpath := dst.GetChanges(id, as, false)
 		bestList = append(bestList, best)
 		oldList = append(oldList, old)
 		if mpath != nil {
 			mpathList = append(mpathList, mpath)
 		}
+
+		if oldmpath != nil {
+			oldmpathList = append(oldmpathList, oldmpath)
+		}
 	}
-	return bestList, oldList, mpathList
+	return bestList, oldList, mpathList, oldmpathList
 }
 
 func (s *BgpServer) propagateUpdateToNeighbors(source *peer, newPath *table.Path, dsts []*table.Update, needOld bool) {
@@ -1303,9 +1315,10 @@ func (s *BgpServer) propagateUpdateToNeighbors(source *peer, newPath *table.Path
 	}
 	var gBestList, gOldList, bestList, oldList []*table.Path
 	var mpathList [][]*table.Path
+	var oldmpathList [][]*table.Path
 	if source == nil || !source.isRouteServerClient() {
-		gBestList, gOldList, mpathList = dstsToPaths(table.GLOBAL_RIB_NAME, 0, dsts)
-		s.notifyBestWatcher(gBestList, mpathList)
+		gBestList, gOldList, mpathList, oldmpathList = dstsToPaths(table.GLOBAL_RIB_NAME, 0, dsts)
+		s.notifyBestWatcher(gBestList, mpathList, oldmpathList)
 	}
 	family := newPath.GetRouteFamily()
 	for _, targetPeer := range s.neighborMap {
@@ -1360,7 +1373,7 @@ func (s *BgpServer) propagateUpdateToNeighbors(source *peer, newPath *table.Path
 				}
 				continue
 			}
-			bestList, oldList, _ = dstsToPaths(targetPeer.TableID(), targetPeer.AS(), dsts)
+			bestList, oldList, _, _ = dstsToPaths(targetPeer.TableID(), targetPeer.AS(), dsts)
 		} else {
 			bestList = gBestList
 			oldList = gOldList
@@ -4297,9 +4310,10 @@ type watchEventTable struct {
 }
 
 type watchEventBestPath struct {
-	PathList      []*table.Path
-	MultiPathList [][]*table.Path
-	Vrf           map[uint32]bool
+	PathList         []*table.Path
+	MultiPathList    [][]*table.Path
+	OldMultiPathList [][]*table.Path
+	Vrf              map[uint32]bool
 }
 
 type watchEventMessage struct {

pkg/server/server_test.go

@@ -749,7 +749,7 @@ func process(rib *table.TableManager, l []*table.Path) (*table.Path, *table.Path
 	for _, path := range l {
 		dsts = append(dsts, rib.Update(path)...)
 	}
-	news, olds, _ := dstsToPaths(table.GLOBAL_RIB_NAME, 0, dsts)
+	news, olds, _, _ := dstsToPaths(table.GLOBAL_RIB_NAME, 0, dsts)
 	if len(news) != 1 {
 		panic("can't handle multiple paths")
 	}
@@ -773,7 +773,7 @@ func TestFilterpathWitheBGP(t *testing.T) {
 	path2 := table.NewPath(pi2, nlri, false, pa2, time.Now(), false)
 	rib.Update(path2)
 	d := rib.Update(path1)
-	new, old, _ := d[0].GetChanges(table.GLOBAL_RIB_NAME, 0, false)
+	new, old, _, _ := d[0].GetChanges(table.GLOBAL_RIB_NAME, 0, false)
 	assert.Equal(t, new, path1)
 	filterpath(p1, new, old)
 	filterpath(p2, new, old)