diff --git a/tests/data/duplicate-code-list/variable/duplicate_entries.yaml b/tests/data/duplicate-code-list/variable/duplicate_entries.yaml
new file mode 100644
index 00000000..0c945f24
--- /dev/null
+++ b/tests/data/duplicate-code-list/variable/duplicate_entries.yaml
@@ -0,0 +1,12 @@
+- Some Variable:
+    description:
+    unit:
+- Some Variable:
+    description:
+    unit:
+- Some other Variable:
+    description:
+    unit:
+- Some other Variable:
+    description:
+    unit:
diff --git a/tests/test_codelist.py b/tests/test_codelist.py
index 5668bc65..1c91ed24 100644
--- a/tests/test_codelist.py
+++ b/tests/test_codelist.py
@@ -1,6 +1,7 @@
 from pytest import raises
 import pandas as pd
 import pandas.testing as pdt
+import pytest
 import logging
 
 from nomenclature.code import Code, RegionCode, MetaCode
@@ -331,11 +332,20 @@ def test_multiple_external_repos():
         TEST_DATA_DIR / "nomenclature_configs" / "multiple_repos_per_dimension.yaml"
     )
     try:
-        with raises(ValueError, match="Duplicate"):
-            variable_code_list = VariableCodeList.from_directory(
+        with raises(ValueError, match="Collected 529 errors"):
+            VariableCodeList.from_directory(
                 "variable",
                 TEST_DATA_DIR / "nomenclature_configs" / "variable",
                 nomenclature_config,
             )
     finally:
         clean_up_external_repos(nomenclature_config.repositories)
+
+
+@pytest.mark.parametrize("CodeList", [VariableCodeList, CodeList])
+def test_variable_codelist_with_duplicates_raises(CodeList):
+    error_string = "2 errors:\n.*Some Variable\n.*Some other Variable"
+    with raises(ValueError, match=error_string):
+        CodeList.from_directory(
+            "variable", TEST_DATA_DIR / "duplicate-code-list" / "variable"
+        )