Merge pull request #51 from jurraca/merge-test-fixup

merge_test: use networks sets instead of counts, updated comments

tests/data/base_file.csv

@@ -1,20 +1,21 @@
-network,is_valid,is_subnet_of_extra
-127.0.0.0/17,FALSE,FALSE
-0.128.0.0/10,TRUE,FALSE
-206.0.0.0/8,TRUE,FALSE
-105.231.36.0/23,TRUE,FALSE
-108.232.0.0/13,TRUE,FALSE
-110.204.32.0/21,TRUE,FALSE
-113.53.128.0/17,TRUE,FALSE
-121.128.0.0/9,TRUE,FALSE
-122.64.0.0/11,TRUE,FALSE
-131.128.0.0/9,TRUE,FALSE
-138.232.0.0/13,TRUE,FALSE
-140.210.168.0/22,TRUE,FALSE
-145.224.0.0/13,TRUE,FALSE
-152.128.0.0/9,TRUE,FALSE
-162.55.64.0/18,TRUE,FALSE
-165.128.0.0/13,TRUE,FALSE
-174.160.0.0/11,TRUE,FALSE
-109.92.213.0/24,TRUE,FALSE
-244.20.0.0/14,TRUE,FALSE
+network,in_result,comment
+127.0.0.0/17,1,merge does not handle sorting out localhost
+0.128.0.0/10,1,private network
+206.0.0.0/8,1,
+105.231.36.0/23,1,
+108.232.0.0/13,1,
+110.204.32.0/21,1,
+113.53.128.0/17,1,
+121.128.0.0/9,1,
+122.64.0.0/11,1,
+131.128.0.0/9,1,
+138.232.0.0/13,1,
+140.210.168.0/22,1,
+145.224.0.0/13,1,
+152.128.0.0/9,1,
+162.55.64.0/18,1,
+165.128.0.0/13,1,
+174.160.0.0/11,1,
+109.92.213.0/24,1,
+244.20.0.0/14,1,
+244.20.0.0/14,1,duplicates within each file are not expected so merge does not handle this

tests/data/extra_file.csv

@@ -1,21 +1,22 @@
-network,is_valid,is_subnet_of_base
-2.11.76.0/22,TRUE,FALSE
-115.7.192.0/19,TRUE,FALSE
-12.132.80.0/20,TRUE,FALSE
-82.64.0.0/11,TRUE,FALSE
-194.48.0.0/17,TRUE,FALSE
-197.216.136.0/21,TRUE,FALSE
-249.0.0.0/9,TRUE,FALSE
-32.17.64.0/20,TRUE,FALSE
-38.66.0.0/17,TRUE,FALSE
-40.240.124.0/22,TRUE,FALSE
-44.26.0.0/19,TRUE,FALSE
-46.99.64.0/20,TRUE,FALSE
-55.48.0.0/15,TRUE,FALSE
-74.0.0.0/10,TRUE,FALSE
-77.120.136.0/22,TRUE,FALSE
-131.128.58.0/23,TRUE,TRUE
-244.20.240.0/20,TRUE,TRUE
-108.232.0.0/16,TRUE,TRUE
-122.68.0.0/14,TRUE,TRUE
-121.128.0.0/10,TRUE,TRUE
+network,in_result,comment
+2.11.76.0/22,1,
+115.7.192.0/19,1,
+12.132.80.0/20,1,
+82.64.0.0/11,1,
+194.48.0.0/17,1,
+197.216.136.0/21,1,
+249.0.0.0/9,1,
+32.17.64.0/20,1,
+38.66.0.0/17,1,
+40.240.124.0/22,1,
+44.26.0.0/19,1,
+46.99.64.0/20,1,
+55.48.0.0/15,1,
+74.0.0.0/10,1,
+77.120.136.0/22,1,
+131.128.58.0/23,0,subnet of 131.128.0.0/9 in base file
+244.20.240.0/20,0,subnet of 244.20.0.0/14 in base file
+108.232.0.0/16,0,subnet of 108.232.0.0/13 in base file
+122.68.0.0/14,0,subnet of 122.64.0.0/11 in base file
+121.128.0.0/10,0,subnet of 121.128.0.0/9 in base file
+109.92.213.0/24,0,duplicate of an entry in base file

tests/test_merge.py

@@ -1,7 +1,7 @@
-"""
+'''
 Test merging multiple sets of networks, as if they were independent AS files.
-"""
-import os
+'''
+from pathlib import Path
 
 from kartograf.merge import general_merge
 
@@ -19,30 +19,31 @@ def __tmp_paths(tmp_path):
     return [tmp_path / p for p in ["rpki_final.txt", "irr_final.txt", "out.txt"]]
 
 def __read_test_vectors(filepath):
-    """
+    '''
     Fixtures for IP networks are under tests/data.
-    Read them and return the list of valid networks and the count of individual subnets expected in the result of the merge.
-    """
-    networks = []
-    subnet_count = 0
+    Read them and return the list of valid networks and the list of individual subnets in the file.
+    '''
+    all_networks = []
+    all_in_result = []
     with open(filepath, "r") as f:
         lines = f.readlines()[1:]
         for line in lines:
-            network, is_valid, is_subnet = line.split(',')
-            if is_valid == "TRUE":
-                networks.append(network)
-            if is_subnet.strip() == "TRUE":
-                subnet_count += 1
-    return networks, subnet_count
+            network, in_result, _comment = line.split(',')
+            all_networks.append(network)
+            if in_result.strip() == "1":
+                all_in_result.append(network)
+    return all_networks, all_in_result
 
 def test_merge_from_fixtures(tmp_path):
     '''
-    Assert that general_merge merges subnets correctly.
+    Assert that general_merge merges subnets correctly,
+    and validates against expected networks,
+    i.e. subnets are merged into the root network appropriately.
     '''
-    data_dir = os.path.join(os.path.dirname(__file__), "data")
-    base_nets, base_subnet_count = __read_test_vectors(os.path.join(data_dir, "base_file.csv"))
+    testdir = Path(__file__).parent
+    base_nets, base_results = __read_test_vectors(testdir / "data/base_file.csv")
     base_path = tmp_path / "base.txt"
-    extra_nets, extra_subnet_count = __read_test_vectors(os.path.join(data_dir, "extra_file.csv"))
+    extra_nets, extra_results = __read_test_vectors(testdir / "data/extra_file.csv")
     extra_path = tmp_path / "extra.txt"
     # write the networks to disk, generating ASNs for each network
     generate_ip_file(base_path, build_file_lines(base_nets, generate_asns(len(base_nets))))
@@ -52,15 +53,16 @@ def test_merge_from_fixtures(tmp_path):
     general_merge(base_path, extra_path, None, outpath)
     with open(outpath, "r") as f:
         l = f.readlines()
-        final_network_count = len(l)
-        expected_count = (len(base_nets) + len(extra_nets)) - (base_subnet_count + extra_subnet_count)
-        assert final_network_count == expected_count
-
+        merged_networks = sorted([line.split()[0] for line in l])
+    expected_networks = sorted(base_results + extra_results)
+    # Compare the merged result of networks, excluding duplicates and subnets,
+    # with the expected results
+    assert merged_networks == expected_networks
 
 def test_merge(tmp_path):
-    """
+    '''
     Assert that merging two identical files is a no-op.
-    """
+    '''
     rpki_data = generate_file_items(100)
     rpki_path, _, out_path = __tmp_paths(tmp_path)
     generate_ip_file(rpki_path, rpki_data)
@@ -75,9 +77,9 @@ def test_merge(tmp_path):
 
 
 def test_merge_disjoint(tmp_path):
-    """
+    '''
     Test merging non-overlapping sets of IP networks.
-    """
+    '''
     main_data = generate_file_items(100)
     main_ips = [item.split()[0] for item in main_data]
     rpki_ips = main_ips[:50]
@@ -99,9 +101,9 @@ def test_merge_disjoint(tmp_path):
 
 
 def test_merge_joint(tmp_path):
-    """
+    '''
     Test merging overlapping sets of IP networks.
-    """
+    '''
     overlap = 10
     rpki_data = generate_file_items(100)
     rpki_ips = [item.split()[0] for item in rpki_data]