Skip to content

Commit

Permalink
Merge pull request #70 from phudtran/scaled-ep-func-tests
Browse files Browse the repository at this point in the history 
Fix for Cross Net Scaled Endpoint Traffic and Additional Scaled Endpoint Tests
  • Loading branch information
@phudtran
phudtran authored Mar 31, 2020
2 parents e23c648 + 220394f commit 5a2984d
Show file tree
Hide file tree
Showing 31 changed files with 600 additions and 209 deletions.
28 changes: 26 additions & 2 deletions src/xdp/trn_transit_xdp.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,14 @@ static __inline int trn_switch_handle_pkt(struct transit_packet *pkt,
ep = bpf_map_lookup_elem(&endpoints_map, &epkey);

if (!ep) {
/* If the scaled endpoint modify option is present,
make TR route to the inner packet source */
if (pkt->scaled_ep_opt->type == TRN_GNV_SCALED_EP_OPT_TYPE &&
pkt->scaled_ep_opt->scaled_ep_data.msg_type ==
TRN_SCALED_EP_MODIFY)
return trn_router_handle_pkt(pkt, inner_dst_ip,
inner_src_ip);

return trn_router_handle_pkt(pkt, inner_src_ip, inner_dst_ip);
}

Expand Down Expand Up @@ -326,6 +334,7 @@ static __inline int trn_handle_scaled_ep_modify(struct transit_packet *pkt)
bpf_ntohl(pkt->scaled_ep_opt->scaled_ep_data.target.daddr));

__be64 tunnel_id = trn_vni_to_tunnel_id(pkt->geneve->vni);

struct scaled_endpoint_remote_t out_tuple;
struct ipv4_tuple_t in_tuple;

Expand Down Expand Up @@ -395,6 +404,10 @@ static __inline int trn_process_inner_ip(struct transit_packet *pkt)
return XDP_ABORTED;
}

/* For whatever compiler reason, we need to perform reverse flow modification
in this function instead of trn_switch_handle_pkt so we keep the orig_src_ip */
__u32 orig_src_ip = pkt->inner_ip->saddr;

pkt->inner_ipv4_tuple.saddr = pkt->inner_ip->saddr;
pkt->inner_ipv4_tuple.daddr = pkt->inner_ip->daddr;
pkt->inner_ipv4_tuple.protocol = pkt->inner_ip->protocol;
Expand Down Expand Up @@ -430,9 +443,21 @@ static __inline int trn_process_inner_ip(struct transit_packet *pkt)
pkt->inner_ipv4_tuple.dport = pkt->inner_udp->dest;
}

/* Lookup the source endpoint*/
__be64 tunnel_id = trn_vni_to_tunnel_id(pkt->geneve->vni);
struct endpoint_t *src_ep;
struct endpoint_key_t src_epkey;

__builtin_memcpy(&src_epkey.tunip[0], &tunnel_id, sizeof(tunnel_id));
src_epkey.tunip[2] = pkt->inner_ip->saddr;
src_ep = bpf_map_lookup_elem(&endpoints_map, &src_epkey);

/* If this is not the source endpoint's host,
skip reverse flow modification, or scaled endpoint modify handling */
if (pkt->scaled_ep_opt->type == TRN_GNV_SCALED_EP_OPT_TYPE &&
pkt->scaled_ep_opt->scaled_ep_data.msg_type ==
TRN_SCALED_EP_MODIFY) {
TRN_SCALED_EP_MODIFY &&
src_ep && src_ep->hosted_iface != -1) {
return trn_handle_scaled_ep_modify(pkt);
}

Expand All @@ -443,7 +468,6 @@ static __inline int trn_process_inner_ip(struct transit_packet *pkt)
sizeof(struct ipv4_tuple_t));

inner_mod = bpf_map_lookup_elem(&rev_flow_mod_cache, &inner);
__u32 orig_src_ip = pkt->inner_ip->saddr;
if (inner_mod) {
/* Modify the inner packet accordingly */
trn_set_src_dst_inner_ip_csum(pkt, inner_mod->saddr,
Expand Down
35 changes: 20 additions & 15 deletions test/trn_controller/endpoint.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ def __init__(self, vni, netid, ip, vpc_cidr, net_cidr, gw_ip, host, tuntype='gnv
if host_ep:
self.veth_name = "host_" + self.veth_name
self.veth_peer = "host_" + self.veth_peer
self.saved_ns = self.ns
self.ns = ""
self.ns_exec = ""
self.backends = backends
Expand All @@ -85,7 +86,7 @@ def __init__(self, vni, netid, ip, vpc_cidr, net_cidr, gw_ip, host, tuntype='gnv
self.udp_server_cmd = f'''{self.bash_cmd} '(nc -u -l -p 5001 > {self.udp_recv_file})' '''
self.iperf3_server_cmd = f'''{self.bash_cmd} 'iperf3 -s > /tmp/{self.ns}_{self.ip}/iperf_server.log 2>&1' '''

self.tcp_serv_idle_cmd = f'''{self.bash_cmd} './mnt/Transit/tools/tcp_server.py' '''
self.tcp_serv_idle_cmd = f'''{self.bash_cmd} './mnt/Transit/tools/tcp_server.py '''
self.tcp_client_idle_cmd = f'''{self.bash_cmd} './mnt/Transit/tools/tcp_client.py '''

def provision(self):
Expand All @@ -109,29 +110,33 @@ def provision(self):
def unprovision(self):
if self.tuntype == 'vxn':
return

if self.host is None:
veth_allocator.getInstance().reclaim_veth(
self.mac, self.ns.replace(self.tuntype + '_', ''), self.veth_peer.replace(self.tuntype + '_', ''))
self.veth_allocated = False
return
else:
if self.host is not None:
self.transit_agent = None
self.host.delete_ep(self)
self.host.unprovision_simple_endpoint(self)

if self.ns == "": # Host endpoint case
veth_allocator.getInstance().reclaim_veth(
self.mac, self.saved_ns.replace(self.tuntype + '_', ''), self.veth_peer.replace("host_" + self.tuntype + '_', ''))
else:
veth_allocator.getInstance().reclaim_veth(
self.mac, self.ns.replace(self.tuntype + '_', ''), self.veth_peer.replace(self.tuntype + '_', ''))
self.veth_allocated = False

self.veth_allocated = False
self.ready = False

def __del__(self):
if (self.tuntype == 'gnv' and self.host):
if self.transit_agent is not None:
self.host.unload_transit_agent_xdp(self.veth_peer)
if self.veth_allocated:
veth_allocator.getInstance().reclaim_veth(
self.mac, self.ns.replace(self.tuntype + '_', ''), self.veth_peer.replace(self.tuntype + '_', ''))
if self.ns == "": # Host endpoint case
veth_allocator.getInstance().reclaim_veth(
self.mac, self.saved_ns.replace(self.tuntype + '_', ''), self.veth_peer.replace("host_" + self.tuntype + '_', ''))
else:
veth_allocator.getInstance().reclaim_veth(
self.mac, self.ns.replace(self.tuntype + '_', ''), self.veth_peer.replace(self.tuntype + '_', ''))

self.veth_allocated = False

def get_tunnel_id(self):
Expand Down Expand Up @@ -249,8 +254,8 @@ def do_iperf3_client(self, ip, args=''):
cmd = f'''{self.bash_cmd} 'iperf3 -c {ip} {args}' '''
return self.host.run(cmd)

def do_tcp_serve_idle(self, detach=True):
return self.host.run(self.tcp_serv_idle_cmd, detach=detach)
def do_tcp_serve_idle(self, connections, idle=0, detach=True):
return self.host.run(self.tcp_serv_idle_cmd + " " + str(connections) + " " + str(idle) + "'", detach=detach)

def do_tcp_client_idle(self, ip, message_count=2, delay=5, detach=True):
return self.host.run(self.tcp_client_idle_cmd + str(ip) + " " + str(message_count) + " " + str(delay) + "'", detach=detach)
def do_tcp_client_idle(self, ip, connection, message_count=2, delay=5, detach=True):
return self.host.run(self.tcp_client_idle_cmd + str(ip) + " " + str(message_count) + " " + str(delay) + " " + str(connection) + "'", detach=detach)
32 changes: 21 additions & 11 deletions test/trn_func_tests/helper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -378,13 +378,14 @@ def do_test_scaled_ep_same_conn(test, client, server, backend):
"{} Testing scaled endpoint same TCP connection multiple messages! {}".format('='*20, '='*20))
exp_pkt_count = 3
ep_to_host_map = {}
total_connections = 1
do_start_pcap_scaled_ep(test, client, server, backend, 13)
for ep in backend:
ep.do_tcp_serve_idle()
ep.do_tcp_serve_idle(total_connections, 1)
ep_to_host_map[ep.host.ip] = ep.ip
client.do_tcp_client_idle(
server.ip, exp_pkt_count, 5)
sleep(13)
server.ip, total_connections, exp_pkt_count, 5)
sleep(15)
backend_packets = {}
for ep in backend:
backend_packets[ep.host.ip] = (rdpcap("test/trn_func_tests/output/" + ep.host.ip + "_" +
Expand All @@ -407,22 +408,25 @@ def do_test_scaled_ep_same_conn(test, client, server, backend):
def do_test_scaled_ep_diff_conn(test, client, server, backend):
"""
This functions validates the scaled endpoint behavior when multiple TCP connections are opened.
1. Opens three TCP connection between the client endpoint and the scaled endpoint.
2. Sends a "Hello, World!" message on all three connections.
1. Opens TCP connection between the client endpoint and the scaled endpoint.
2. Sends a "Hello, World!" message on all connections.
3. Validates that at least TWO hosts in the backend recieved the the "Hello, World!" messages.
"""
logger.info(
"{} Testing scaled endpoint multiple TCP connections! {}".format('='*20, '='*20))
total_connections = 10
total_connections = 50
ep_to_host_map = {}
host_pkt_distrib = {}
do_start_pcap_scaled_ep(test, client, server, backend, 10)
for ep in backend:
ep.do_tcp_serve_idle()
ep.do_tcp_serve_idle(total_connections)
ep_to_host_map[ep.host.ip] = ep.ip
for __ in range(0, total_connections):
client.do_tcp_client_idle(
server.ip, 1, 1)
sleep(10)
host_pkt_distrib[ep.host.ip] = 0
logger.info(
"{} Backend Host: {} IP: {} {}".format('='*20, ep.host.id, ep.host.ip, '='*20))
for conn in range(0, total_connections):
client.do_tcp_client_idle(server.ip, conn)
sleep(15)
backend_packets = {}
for ep in backend:
backend_packets[ep.host.ip] = (rdpcap("test/trn_func_tests/output/" + ep.host.ip + "_" +
Expand All @@ -435,6 +439,12 @@ def do_test_scaled_ep_diff_conn(test, client, server, backend):
message = pkt[GENEVE][TCP][Raw].load.decode()
if message == "Hello, World!" and pkt[GENEVE][IP].dst == ep_to_host_map[host_ip]:
diff_host_count[host_ip] = 1
host_pkt_distrib[host_ip] += 1
logger.info(
"{} {} Backend host(s) recieved a message! {}".format('='*20, sum(diff_host_count.values()), '='*20))
for host in host_pkt_distrib.keys():
logger.info(
"{} Backend Host {}: recieved {} messages! {}".format('='*20, host, host_pkt_distrib[host], '='*20))
test.assertTrue(sum(diff_host_count.values()) > 1)


Expand Down
16 changes: 8 additions & 8 deletions test/trn_func_tests/test_basic_scaled_endpoint.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,16 +59,16 @@ def setUp(self):
"d4": droplet("d4"),
"d5": droplet("d5")
}
c = controller(self.droplets)
self.c = controller(self.droplets)

c.create_vpc(3, cidr("16", "10.0.0.0"), [])
c.create_network(3, 1, cidr("16", "10.0.0.0"), ["d4"])
self.ep0 = c.create_simple_endpoint(3, 1, "10.0.0.2", "d0")
self.ep1 = c.create_simple_endpoint(3, 1, "10.0.0.3", "d1")
self.ep2 = c.create_simple_endpoint(3, 1, "10.0.0.4", "d2")
self.ep3 = c.create_simple_endpoint(3, 1, "10.0.0.5", "d3")
self.c.create_vpc(3, cidr("16", "10.0.0.0"), [])
self.c.create_network(3, 1, cidr("16", "10.0.0.0"), ["d4"])
self.ep0 = self.c.create_simple_endpoint(3, 1, "10.0.0.2", "d0")
self.ep1 = self.c.create_simple_endpoint(3, 1, "10.0.0.3", "d1")
self.ep2 = self.c.create_simple_endpoint(3, 1, "10.0.0.4", "d2")
self.ep3 = self.c.create_simple_endpoint(3, 1, "10.0.0.5", "d3")

self.sep = c.create_scaled_endpoint(3, 1, "10.0.0.6", [self.ep2])
self.sep = self.c.create_scaled_endpoint(3, 1, "10.0.0.6", [self.ep2])

def tearDown(self):
pass
Expand Down
11 changes: 6 additions & 5 deletions test/trn_func_tests/test_basic_switch.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,12 +56,13 @@ def setUp(self):
"right": droplet("right"),
"switch": droplet("switch"),
}
c = controller(self.droplets)
self.c = controller(self.droplets)

c.create_vpc(3, cidr("16", "10.0.0.0"), [])
c.create_network(3, 1, cidr("16", "10.0.0.0"), ["switch"])
self.ep_left = c.create_simple_endpoint(3, 1, "10.0.0.2", "left")
self.ep_right = c.create_simple_endpoint(3, 1, "10.0.0.3", "right")
self.c.create_vpc(3, cidr("16", "10.0.0.0"), [])
self.c.create_network(3, 1, cidr("16", "10.0.0.0"), ["switch"])
self.ep_left = self.c.create_simple_endpoint(3, 1, "10.0.0.2", "left")
self.ep_right = self.c.create_simple_endpoint(
3, 1, "10.0.0.3", "right")

def tearDown(self):
pass
Expand Down
13 changes: 7 additions & 6 deletions test/trn_func_tests/test_delete_endpoint.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,17 +56,18 @@ def setUp(self):
"right": droplet("right"),
"switch": droplet("switch"),
}
c = controller(self.droplets)
self.c = controller(self.droplets)

c.create_vpc(3, cidr("16", "10.0.0.0"), [])
c.create_network(3, 1, cidr("16", "10.0.0.0"), ["switch"])
self.ep_right = c.create_simple_endpoint(3, 1, "10.0.0.3", "right")
self.c.create_vpc(3, cidr("16", "10.0.0.0"), [])
self.c.create_network(3, 1, cidr("16", "10.0.0.0"), ["switch"])
self.ep_right = self.c.create_simple_endpoint(
3, 1, "10.0.0.3", "right")
# Clear all droplet update state before we create and delete ep_left
self.droplets["switch"].clear_update_call_state()
self.droplets["right"].clear_update_call_state()

self.ep_left = c.create_simple_endpoint(3, 1, "10.0.0.2", "left")
self.ep_left = c.delete_simple_endpoint(3, 1, "10.0.0.2")
self.ep_left = self.c.create_simple_endpoint(3, 1, "10.0.0.2", "left")
self.ep_left = self.c.delete_simple_endpoint(3, 1, "10.0.0.2")

def tearDown(self):
pass
Expand Down
20 changes: 11 additions & 9 deletions test/trn_func_tests/test_delete_multi_endpoint_same_host.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,18 +56,20 @@ def setUp(self):
"right": droplet("right"),
"switch": droplet("switch"),
}
c = controller(self.droplets)
self.c = controller(self.droplets)

c.create_vpc(3, cidr("16", "10.0.0.0"), [])
c.create_network(3, 1, cidr("16", "10.0.0.0"), ["switch"])
self.ep_right = c.create_simple_endpoint(3, 1, "10.0.0.3", "right")
self.ep_left = c.create_simple_endpoint(3, 1, "10.0.0.2", "left")
self.c.create_vpc(3, cidr("16", "10.0.0.0"), [])
self.c.create_network(3, 1, cidr("16", "10.0.0.0"), ["switch"])
self.ep_right = self.c.create_simple_endpoint(
3, 1, "10.0.0.3", "right")
self.ep_left = self.c.create_simple_endpoint(3, 1, "10.0.0.2", "left")

self.ep_right2 = c.create_simple_endpoint(3, 1, "10.0.0.4", "right")
self.ep_left2 = c.create_simple_endpoint(3, 1, "10.0.0.5", "left")
self.ep_right2 = self.c.create_simple_endpoint(
3, 1, "10.0.0.4", "right")
self.ep_left2 = self.c.create_simple_endpoint(3, 1, "10.0.0.5", "left")

self.ep_left = c.delete_simple_endpoint(3, 1, "10.0.0.2")
self.ep_right = c.delete_simple_endpoint(3, 1, "10.0.0.3")
self.ep_left = self.c.delete_simple_endpoint(3, 1, "10.0.0.2")
self.ep_right = self.c.delete_simple_endpoint(3, 1, "10.0.0.3")

def tearDown(self):
pass
Expand Down
18 changes: 9 additions & 9 deletions test/trn_func_tests/test_delete_net.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,26 +80,26 @@ def setUp(self):
"router-1": droplet("router-1")
}

c = controller(self.droplets)
self.c = controller(self.droplets)

c.create_vpc(3, cidr("16", "10.0.0.0"), ["router-1"])
c.create_network(3, 10, cidr("24", "10.0.0.0"), ["switch-1"])
self.c.create_vpc(3, cidr("16", "10.0.0.0"), ["router-1"])
self.c.create_network(3, 10, cidr("24", "10.0.0.0"), ["switch-1"])

# Clear update information about "10.0.0.0" subnet
# When comparing update and delete RPC calls, we don't delete subnet "10.0.0.0".
# Thus, there is no corresponding delete_net in the droplet's rpc_deletes map
self.droplets["router-1"].clear_update_call_state()

c.create_network(3, 20, cidr("24", "10.20.0.0"), ["switch-2"])
self.c.create_network(3, 20, cidr("24", "10.20.0.0"), ["switch-2"])

self.ep1 = c.create_simple_endpoint(3, 10, "10.0.0.2", "d1")
self.ep2 = c.create_simple_endpoint(3, 10, "10.0.0.3", "d2")
self.ep1 = self.c.create_simple_endpoint(3, 10, "10.0.0.2", "d1")
self.ep2 = self.c.create_simple_endpoint(3, 10, "10.0.0.3", "d2")

self.ep3 = c.create_simple_endpoint(3, 20, "10.20.0.4", "d3")
self.ep4 = c.create_simple_endpoint(3, 20, "10.20.0.5", "d4")
self.ep3 = self.c.create_simple_endpoint(3, 20, "10.20.0.4", "d3")
self.ep4 = self.c.create_simple_endpoint(3, 20, "10.20.0.5", "d4")

# Delete the network and its corresponding endpoints
c.delete_network(3, 20)
self.c.delete_network(3, 20)

def tearDown(self):
pass
Expand Down
22 changes: 11 additions & 11 deletions test/trn_func_tests/test_delete_one_at_a_time.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,22 +80,22 @@ def setUp(self):
"router-1": droplet("router-1")
}

c = controller(self.droplets)
self.c = controller(self.droplets)

c.create_vpc(3, cidr("16", "10.0.0.0"), ["router-1"])
c.create_network(3, 10, cidr("24", "10.0.0.0"), ["switch-1"])
c.create_network(3, 20, cidr("24", "10.20.0.0"), ["switch-2"])
self.c.create_vpc(3, cidr("16", "10.0.0.0"), ["router-1"])
self.c.create_network(3, 10, cidr("24", "10.0.0.0"), ["switch-1"])
self.c.create_network(3, 20, cidr("24", "10.20.0.0"), ["switch-2"])

self.ep1 = c.create_simple_endpoint(3, 10, "10.0.0.2", "d1")
self.ep2 = c.create_simple_endpoint(3, 10, "10.0.0.3", "d2")
self.ep1 = self.c.create_simple_endpoint(3, 10, "10.0.0.2", "d1")
self.ep2 = self.c.create_simple_endpoint(3, 10, "10.0.0.3", "d2")

self.ep3 = c.create_simple_endpoint(3, 20, "10.20.0.4", "d3")
self.ep4 = c.create_simple_endpoint(3, 20, "10.20.0.5", "d4")
self.ep3 = self.c.create_simple_endpoint(3, 20, "10.20.0.4", "d3")
self.ep4 = self.c.create_simple_endpoint(3, 20, "10.20.0.5", "d4")

# Delete an endpoint in subnet, delete the parent subnet, then parent VPC.
self.ep3 = c.delete_simple_endpoint(3, 20, "10.20.0.4")
c.delete_network(3, 20)
c.delete_vpc(3)
self.ep3 = self.c.delete_simple_endpoint(3, 20, "10.20.0.4")
self.c.delete_network(3, 20)
self.c.delete_vpc(3)

def tearDown(self):
pass
Expand Down
Loading

0 comments on commit 5a2984d

Please sign in to comment.