Merge branch 'fix/pfxdedup' of github.com:bio-routing/bio-rd into fix/pfxdedup

AUTHORS

@@ -13,5 +13,6 @@
 Cedric Kienzler
 Christoph Petrausch
 Daniel Czerwonk
+Maximilian Wilhelm
 Oliver Herms
 Serge Bazanski

CONTRIBUTORS

@@ -9,5 +9,6 @@ Cedric Kienzler
 Christoph Petrausch
 Daniel Czerwonk
 Julian Kornberger
+Maximilian Wilhelm
 Oliver Herms
 Serge Bazanski

RFCs.md

+# List of RFCs bio routing implements
+
+Please keep this list ordered.
+
+ * 1997 BGP Communities Attribute
+ * 4271 A Border Gateway Protocol 4 (BGP-4)
+ * 4456 BGP Route Reflection
+ * 4760 Multiprotocol Extensions for BGP-4
+ * 6793 32bit ASNs
+ * 7911 BGP AddPath
+ * 7947 BGP Route Server
+ * 8092 BGP Large Communities Attribute
+ * 8212 Default External BGP (EBGP) Route Propagation Behavior without Policies

benchmarks/ipcache/main.go

+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"runtime/pprof"
+	"time"
+
+	bnet "github.com/bio-routing/bio-rd/net"
+)
+
+func main() {
+	for i := 0; i < 255; i++ {
+		for j := 0; j < 255; j++ {
+			for k := 0; k < 11; k++ {
+				addr := bnet.IPv4FromOctets(10, uint8(i), uint8(j), uint8(k))
+				addr.Dedup()
+			}
+		}
+	}
+
+	buf := bytes.NewBuffer(nil)
+	err := pprof.StartCPUProfile(buf)
+	if err != nil {
+		panic(err)
+	}
+
+	start := time.Now().UnixNano()
+
+	for x := 0; x < 1; x++ {
+		for i := 0; i < 255; i++ {
+			for j := 0; j < 255; j++ {
+				for k := 0; k < 11; k++ {
+					addr := bnet.IPv4FromOctets(10, uint8(i), uint8(j), uint8(k))
+					addr.Dedup()
+				}
+			}
+		}
+	}
+
+	end := time.Now().UnixNano()
+
+	d := end - start
+	pprof.StopCPUProfile()
+	fmt.Printf("Looking up IP-Addresses took %d ms\n", d/1000000)
+
+	ioutil.WriteFile("profile.pprof", buf.Bytes(), 0644)
+
+	x := bytes.NewBuffer(nil)
+	pprof.WriteHeapProfile(x)
+
+	ioutil.WriteFile("heap.pprof", x.Bytes(), 0644)
+}

benchmarks/pfxcache/main.go

+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"runtime/pprof"
+	"time"
+
+	bnet "github.com/bio-routing/bio-rd/net"
+)
+
+func main() {
+	pfxs := make([]*bnet.Prefix, 0)
+	for i := 0; i < 255; i++ {
+		for j := 0; j < 255; j++ {
+			for k := 0; k < 11; k++ {
+				addr := bnet.IPv4FromOctets(uint8(k)+1, uint8(i), uint8(j), 0)
+				addr.Dedup()
+
+				pfxs = append(pfxs, bnet.NewPfx(addr, 24).Dedup())
+			}
+		}
+	}
+
+	buf := bytes.NewBuffer(nil)
+	err := pprof.StartCPUProfile(buf)
+	if err != nil {
+		panic(err)
+	}
+
+	start := time.Now().UnixNano()
+
+	for i := range pfxs {
+		pfxs[i].Dedup()
+	}
+
+	end := time.Now().UnixNano()
+
+	d := end - start
+	pprof.StopCPUProfile()
+	fmt.Printf("Looking up Prefixes took %d ms\n", d/1000000)
+
+	ioutil.WriteFile("profile.pprof", buf.Bytes(), 0644)
+
+	x := bytes.NewBuffer(nil)
+	pprof.WriteHeapProfile(x)
+
+	ioutil.WriteFile("heap.pprof", x.Bytes(), 0644)
+}

go.mod

@@ -5,6 +5,7 @@ require (
 	github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 // indirect
 	github.com/bio-routing/tflow2 v0.0.0-20181230153523-2e308a4a3c3a
 	github.com/golang/protobuf v1.3.1
+	github.com/google/btree v1.0.0
 	github.com/grpc-ecosystem/go-grpc-middleware v1.0.0
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
 	github.com/pkg/errors v0.8.0
@@ -20,3 +21,5 @@ require (
 	google.golang.org/grpc v1.17.0
 	gopkg.in/yaml.v2 v2.2.2
 )
+
+go 1.13

go.sum

@@ -30,6 +30,8 @@ github.com/golang/protobuf v1.2.1-0.20181128192352-1d3f30b51784 h1:s1jVWjw0DeCSJ
 github.com/golang/protobuf v1.2.1-0.20181128192352-1d3f30b51784/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
 github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=

net/ip.go

@@ -5,6 +5,7 @@ import (
 	"net"
 
 	api "github.com/bio-routing/bio-rd/net/api"
+	"github.com/google/btree"
 )
 
 // IP represents an IPv4 or IPv6 address
@@ -122,12 +123,13 @@ func (ip *IP) Equal(other *IP) bool {
 	return *ip == *other
 }
 
+// Less compares ips for use in btree.Btree
+func (ip *IP) Less(other btree.Item) bool {
+	return ip.Compare(other.(*IP)) == -1
+}
+
 // Compare compares two IP addresses (returns 0 if equal, -1 if `ip` is smaller than `other`, 1 if `ip` is greater than `other`)
 func (ip *IP) Compare(other *IP) int8 {
-	if ip.Equal(other) {
-		return 0
-	}
-
 	if ip.higher > other.higher {
 		return 1
 	}
@@ -140,9 +142,14 @@ func (ip *IP) Compare(other *IP) int8 {
 		return 1
 	}
 
-	return -1
+	if ip.lower < other.lower {
+		return -1
+	}
+
+	return 0
 }
 
+// String returns string representation of an IP address
 func (ip *IP) String() string {
 	if !ip.isLegacy {
 		return ip.stringIPv6()

net/ip_cache.go

 package net
 
-import "sync"
+import (
+	"sync"
+
+	"github.com/google/btree"
+)
 
 const (
-	ipCacheInitialSize = 1000000
+	ipCacheBTreeGrade = 3500
 )
 
 var (
@@ -15,25 +19,27 @@ func init() {
 }
 
 type ipCache struct {
-	cache   map[IP]*IP
 	cacheMu sync.Mutex
+	tree    *btree.BTree
 }
 
 func newIPCache() *ipCache {
 	return &ipCache{
-		cache: make(map[IP]*IP, ipCacheInitialSize),
+		tree: btree.New(ipCacheBTreeGrade),
 	}
 }
 
 func (ipc *ipCache) get(addr *IP) *IP {
 	ipc.cacheMu.Lock()
 
-	if x, ok := ipc.cache[*addr]; ok {
+	item := ipc.tree.Get(addr)
+	if item != nil {
 		ipc.cacheMu.Unlock()
-		return x
+
+		return item.(*IP)
 	}
 
-	ipc.cache[*addr] = addr
+	ipc.tree.ReplaceOrInsert(addr)
 	ipc.cacheMu.Unlock()
 
 	return addr

net/ip_cache_test.go

+package net
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestIPCache(t *testing.T) {
+	a := &IP{
+		higher:   100,
+		lower:    200,
+		isLegacy: false,
+	}
+	b := &IP{
+		higher:   100,
+		lower:    200,
+		isLegacy: false,
+	}
+
+	x := a.Dedup()
+	y := b.Dedup()
+
+	assert.Equal(t, true, x == y)
+}

net/prefix.go

@@ -6,8 +6,10 @@ import (
 	gonet "net"
 	"strconv"
 	"strings"
+	"unsafe"
 
 	"github.com/bio-routing/bio-rd/net/api"
+	"github.com/google/btree"
 	"github.com/pkg/errors"
 )
 
@@ -22,6 +24,18 @@ func (p *Prefix) Dedup() *Prefix {
 	return pfxc.get(p)
 }
 
+<<<<<<< HEAD
+=======
+// Less compares prefixes for use in btree.Btree
+func (p *Prefix) Less(other btree.Item) bool {
+	if uintptr(unsafe.Pointer(p.addr)) < uintptr(unsafe.Pointer(other.(*Prefix).addr)) {
+		return true
+	}
+
+	return p.pfxlen < other.(*Prefix).pfxlen
+}
+
+>>>>>>> 9a2f84b22a4aac17b61e2cdddefd23f228234246
 // DedupWithIP gets a copy of Prefix from the cache and dedups the IP part
 func (p *Prefix) DedupWithIP() *Prefix {
 	p.addr = p.addr.Dedup()

net/prefix_cache.go

 package net
 
-import "sync"
+import (
+	"sync"
+
+	"github.com/google/btree"
+)
 
 const (
-	prefixCacheInitialSize = 1000000
+	prefixCacheBTreeGrade = 3500
 )
 
 var (
@@ -15,25 +19,26 @@ func init() {
 }
 
 type pfxCache struct {
-	cache   map[Prefix]*Prefix
 	cacheMu sync.Mutex
+	tree    *btree.BTree
 }
 
 func newPfxCache() *pfxCache {
 	return &pfxCache{
-		cache: make(map[Prefix]*Prefix, prefixCacheInitialSize),
+		tree: btree.New(prefixCacheBTreeGrade),
 	}
 }
 
 func (pfxc *pfxCache) get(pfx *Prefix) *Prefix {
 	pfxc.cacheMu.Lock()
 
-	if x, ok := pfxc.cache[*pfx]; ok {
+	item := pfxc.tree.Get(pfx)
+	if item != nil {
 		pfxc.cacheMu.Unlock()
-		return x
+		return item.(*Prefix)
 	}
 
-	pfxc.cache[*pfx] = pfx
+	pfxc.tree.ReplaceOrInsert(pfx)
 	pfxc.cacheMu.Unlock()
 
 	return pfx

net/prefix_cache_test.go

+package net
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestPrefixCache(t *testing.T) {
+	a := &Prefix{
+		addr: &IP{
+			higher:   100,
+			lower:    200,
+			isLegacy: false,
+		},
+		pfxlen: 64,
+	}
+	b := &Prefix{
+		addr: &IP{
+			higher:   100,
+			lower:    200,
+			isLegacy: false,
+		},
+		pfxlen: 64,
+	}
+
+	a.addr = a.addr.Dedup()
+	b.addr = b.addr.Dedup()
+
+	x := a.Dedup()
+	y := b.Dedup()
+
+	assert.Equal(t, true, x == y)
+}