DTSRD-3529. According to the changed requirement. (#431)

* DTSRD-3529. add 2 extra columns in the LOV table * DTSRD-3529. add 2 extra columns in the LOV table * DTSRD-3529. add 2 extra columns in the LOV table * DTSRD-3529. add 2 extra columns in the LOV table * DTSRD-3529. add 2 extra columns in the LOV table * DTSRD-3529. add 2 extra columns in the LOV table * DTSRD-3529. add 2 extra columns in the LOV table * DTSRD-3529. add 2 extra columns in the LOV table * DTSRD-3529. add 2 extra columns in the LOV table * DTSRD-3529. add 2 extra columns in the LOV table * DTSRD-3529. add 2 extra columns in the LOV table * DTSRD-3529. add 2 extra columns in the LOV table * DTSRD-3529. add 2 extra columns in the LOV table * DTSRD-3529. add 2 extra columns in the LOV table * DTSRD-3529. add 2 extra columns in the LOV table * DTSRD-3529. add 2 extra columns in the LOV table * DTSRD-3529. add 2 extra columns in the LOV table * DTSRD-3529. add 2 extra columns in the LOV table * DTSRD-3529. add 2 extra columns in the LOV table * DTSRD-3529. changes based on review comments * DTSRD-3529. changes based on review comments * DTSRD-3529. changes based on review comments * DTSRD-3529. changes based on review comments * DTSRD-3529. changes based on review comments * DTSRD-3529. changes based on review comments * DTSRD-3529. changes based on review comments * DTSRD-3529. changes based on review comments * DTSRD-3529. changes based on review comments * DTSRD-3529. changes based on review comments * DTSRD-3529. changes based on review comments * DTSRD-3529. changes based on review comments * DTSRD-3529. changes based on review comments * DTSRD-3529. changes based on review comments * DTSRD-3529. changes based on review comments * DTSRD-3529. changes based on review comments * DTSRD-3529. changes based on review comments * DTSRD-3529. changes to add headers check test case fro external reference * DTSRD-3529. changes to add headers check test case fro external reference * DTSRD-3529. changes to add headers check test case fro external reference
hmcts · Nov 28, 2024 · eb0246a · eb0246a
1 parent 1b56dd8
commit eb0246a
Show file tree

Hide file tree

Showing 40 changed files with 549 additions and 161 deletions.
diff --git a/bin/functionalTest/sourceFiles/categories/list_of_values_utf8_header.csv b/bin/functionalTest/sourceFiles/categories/list_of_values_utf8_header.csv
@@ -1,6 +1,6 @@
-CategoryKey,ServiceID,Key,Value_EN,Value_CY,Hinttext_EN,Hinttext_CY,Lov_Order,ParentCategory,ParentKey,Active
-AdditionalFacilities,,AF-VF,Video Facility,,,,,,,Y
-AdditionalFacilities,,AF-SSC,Same Sex Courtroom,,,,,,,Y
-AdditionalFacilities,,AF-WR,Witness Room,,,,,,,Y
+CategoryKey,ServiceID,Key,Value_EN,Value_CY,Hinttext_EN,Hinttext_CY,Lov_Order,ParentCategory,ParentKey,Active,External_Reference,External_Reference_Type
+AdditionalFacilities,,AF-VF,Video Facility,,,,,,,Y,,
+AdditionalFacilities,,AF-SSC,Same Sex Courtroom,,,,,,,Y,,
+AdditionalFacilities,,AF-WR,Witness Room,,,,,,,Y,,
 
 
diff --git a/bin/main/application-crd-list-of-values-router.yaml b/bin/main/application-crd-list-of-values-router.yaml
@@ -9,17 +9,17 @@ route:
     table-name: list_of_values
     insert-sql:
       sql:insert into list_of_values (categorykey,serviceid,key,value_en,value_cy,hinttext_en,hinttext_cy,lov_order,
-      parentcategory,parentkey,active)
+      parentcategory,parentkey,active,external_reference,external_reference_type)
       values(:#categoryKey,:#serviceId,:#key,:#value_en,:#value_cy,:#hinttext_en,:#hinttext_cy,:#lov_order,
-      :#parentcategory,:#parentkey,:#active) on conflict (categorykey,key,serviceid) do UPDATE SET categorykey = :#categoryKey,
+      :#parentcategory,:#parentkey,:#active,:#external_reference,:#external_reference_type) on conflict (categorykey,key,serviceid) do UPDATE SET categorykey = :#categoryKey,
       serviceid = :#serviceId, value_en = :#value_en, value_cy = :#value_cy, hinttext_en =:#hinttext_en,
       hinttext_cy = :#hinttext_cy, lov_order = :#lov_order, parentcategory = :#parentcategory, parentkey = :#parentkey,
-      active = :#active?batch=true&dataSource=dataSource
+      active = :#active, external_reference = :#external_reference,external_reference_type = :#external_reference_type?batch=true&dataSource=dataSource
     blob-path:
       azure-blob://${azure.storage.account-name}/rd-common-data/ListOfValues.csv?credentials=#credsreg&operation=updateBlockBlob
     processor-class: categoriesProcessor
     mapper-class: categoriesMapper
     csv-binder-object: Categories
-    csv-headers-expected: categorykey,serviceid,key,value_en,value_cy,hinttext_en,hinttext_cy,lov_order,parentcategory,parentkey,active
+    csv-headers-expected: categorykey,serviceid,key,value_en,value_cy,hinttext_en,hinttext_cy,lov_order,parentcategory,parentkey,active,external_reference,external_reference_type
     header-validation-enabled: true
 
diff --git a/...alTest/java/uk/gov/hmcts/reform/rd/commondata/cameltest/CommonDataCategoriesLoadTest.java b/...alTest/java/uk/gov/hmcts/reform/rd/commondata/cameltest/CommonDataCategoriesLoadTest.java
@@ -38,6 +38,7 @@
 import java.util.Map;
 import java.util.Optional;
 
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.springframework.util.ResourceUtils.getFile;
@@ -188,6 +189,30 @@ void testListOfValuesCsv_MissingHeader_Failure() throws Exception {
         validateFlagServiceFileAudit(jdbcTemplate, auditSchedulerQuery, "Failure", UPLOAD_LIST_OF_VALUES_FILE_NAME);
     }
 
+
+    @Test
+    @DisplayName("Status: Failure - Test for missing external reference Upload Filed.")
+    @Sql(scripts = {"/testData/commondata_truncate.sql"})
+    void testListOfValuesExternalReferenceHeadersMissing() throws Exception {
+
+        commonDataBlobSupport.uploadFile(
+            UPLOAD_LIST_OF_VALUES_FILE_NAME,
+            new FileInputStream(getFile(
+                "classpath:sourceFiles/categories/list_of_values_failure_missing_header.csv"))
+        );
+
+        jobLauncherTestUtils.launchJob();
+        var listOfValues = jdbcTemplate.queryForList(listOfValuesSelectData);
+        assertEquals(0, listOfValues.size());
+
+        Pair<String, String> pair = new Pair<>(
+            UPLOAD_LIST_OF_VALUES_FILE_NAME,
+            "There is a mismatch in the headers of the csv file :: ListOfValues-test.csv"
+        );
+        validateFlagServiceFileException(jdbcTemplate, exceptionQuery, pair, 3);
+        validateFlagServiceFileAudit(jdbcTemplate, auditSchedulerQuery, "Failure", UPLOAD_LIST_OF_VALUES_FILE_NAME);
+    }
+
     @Test
     @DisplayName("Status: Sucess - Test for LOV Duplicate records Case1.Filters duplicate records")
     @Sql(scripts = {"/testData/commondata_truncate.sql"})
@@ -313,7 +338,111 @@ void testListOfValuesCsv_0_byte_character() throws Exception {
         validateCategoriesFileAudit(
             jdbcTemplate,
             auditSchedulerQuery,
-            "Failure",
+            "PartialSuccess",
+            UPLOAD_LIST_OF_VALUES_FILE_NAME
+        );
+    }
+
+    @Test
+    @DisplayName("Status: Sucess - all values exist.")
+    @Sql(scripts = {"/testData/commondata_truncate.sql"})
+    void testListOfValuesExternalReferenceSuccess() throws Exception {
+        commonDataBlobSupport.uploadFile(
+            UPLOAD_LIST_OF_VALUES_FILE_NAME,
+            new FileInputStream(getFile(
+                "classpath:sourceFiles/categories/list_of_values_external_reference_success.csv"))
+        );
+
+        jobLauncherTestUtils.launchJob();
+        var listOfValues = jdbcTemplate.queryForList(listOfValuesSelectData);
+        assertEquals(3, listOfValues.size());
+        //Validate Success Result
+        validateListOfValuesFileWithExternalReference(jdbcTemplate, listOfValuesSelectData, List.of(
+            Categories.builder().categoryKey("panelCategoryMember").serviceId("BBA3").key("PC1-01-74")
+                .valueEN("Medical office holder").valueCY("").hintTextEN("").hintTextCY("").parentCategory(
+                    "caseSubType").parentKey("PC2").active("Y").externalReference("74")
+                .externalReferenceType("MedicalRole").build(),
+            Categories.builder().categoryKey("panelCategoryMember").serviceId("BBA3").key("PC1-01-94")
+                .valueEN("Financial office holder").valueCY("").hintTextEN("").hintTextCY("")
+                .parentCategory("caseSubType")
+                .parentKey("PC3").active("Y").externalReference("94").externalReferenceType("FinancialRole").build(),
+            Categories.builder().categoryKey("panelCategoryMember").serviceId("BBA3").key("PC1-01-84")
+                .valueEN("Judicial office holder").valueCY("").hintTextEN("").hintTextCY("")
+                .parentCategory("caseSubType")
+                .parentKey("PC1").active("Y").externalReference("84").externalReferenceType("JudicialRole")
+                .build()), 3);
+        //Validates Success Audit
+        validateFlagServiceFileAudit(jdbcTemplate, auditSchedulerQuery, "Success",
+            UPLOAD_LIST_OF_VALUES_FILE_NAME);
+
+    }
+
+    @Test
+    @DisplayName("Status: Sucess - all values exist.")
+    @Sql(scripts = {"/testData/commondata_truncate.sql"})
+    void testListOfValuesAllRecordsContainNullExternalReferenceTypeAndNullExternalReferenceSuccess() throws Exception {
+        commonDataBlobSupport.uploadFile(
+            UPLOAD_LIST_OF_VALUES_FILE_NAME,
+            new FileInputStream(getFile(
+                "classpath:sourceFiles/categories/list_of_values_external_reference_null_success.csv"))
+        );
+
+        jobLauncherTestUtils.launchJob();
+        var listOfValues = jdbcTemplate.queryForList(listOfValuesSelectData);
+        assertEquals(3, listOfValues.size());
+        //Validate Success Result
+        assertThat(listOfValues.get(0).get("categoryKey")).isEqualTo("panelCategoryMember");
+        assertThat(listOfValues.get(0).get("key")).isEqualTo("PC1-01-74");
+        assertThat(listOfValues.get(0).get("parentcategory")).isEqualTo("caseSubType");
+        assertThat(listOfValues.get(0).get("parentkey")).isEqualTo("PC2");
+        assertThat(listOfValues.get(0).get("external_reference")).isEqualTo("");
+        assertThat(listOfValues.get(0).get("external_reference_type")).isEqualTo("");
+
+        assertThat(listOfValues.get(1).get("categoryKey")).isEqualTo("panelCategoryMember");
+        assertThat(listOfValues.get(1).get("key")).isEqualTo("PC1-01-94");
+        assertThat(listOfValues.get(1).get("parentcategory")).isEqualTo("caseSubType");
+        assertThat(listOfValues.get(1).get("parentkey")).isEqualTo("PC3");
+        assertThat(listOfValues.get(1).get("external_reference")).isEqualTo("");
+        assertThat(listOfValues.get(1).get("external_reference_type")).isEqualTo("");
+
+        assertThat(listOfValues.get(2).get("categoryKey")).isEqualTo("panelCategoryMember");
+        assertThat(listOfValues.get(2).get("key")).isEqualTo("PC1-01-84");
+        assertThat(listOfValues.get(2).get("parentcategory")).isEqualTo("caseSubType");
+        assertThat(listOfValues.get(2).get("parentkey")).isEqualTo("PC1");
+        assertThat(listOfValues.get(2).get("external_reference")).isEqualTo("");
+        assertThat(listOfValues.get(2).get("external_reference_type")).isEqualTo("");
+        //Validates Success Audit
+        validateFlagServiceFileAudit(jdbcTemplate, auditSchedulerQuery, "Success",
+            UPLOAD_LIST_OF_VALUES_FILE_NAME);
+
+    }
+
+    @Test
+    @DisplayName("Status: Failure - Test for either of the external reference values are null.")
+    @Sql(scripts = {"/testData/commondata_truncate.sql"})
+    void testListOfValuesExternalReferenceFailure() throws Exception {
+        commonDataBlobSupport.uploadFile(
+            UPLOAD_LIST_OF_VALUES_FILE_NAME,
+            new FileInputStream(getFile(
+                "classpath:sourceFiles/categories/list_of_values_external_reference_failure.csv"))
+        );
+
+        jobLauncherTestUtils.launchJob();
+        var listOfValues = jdbcTemplate.queryForList(listOfValuesSelectData);
+        assertEquals(1, listOfValues.size());
+
+        String externalReferenceErrorMessage = "Both external_reference and external_reference_type "
+            + "value must be null or both must be not-null";
+
+        Pair<String, String> pair = new Pair<>(
+            UPLOAD_LIST_OF_VALUES_FILE_NAME,
+            externalReferenceErrorMessage
+        );
+        validateCategoriesFileException(jdbcTemplate, exceptionQuery, pair);
+        validateCategoriesFileAudit(
+            jdbcTemplate,
+            auditSchedulerQuery,
+            "PartialSuccess",
             UPLOAD_LIST_OF_VALUES_FILE_NAME
         );
     }

diff --git a/...uk/gov/hmcts/reform/rd/commondata/cameltest/testsupport/CommonDataFunctionalBaseTest.java b/...uk/gov/hmcts/reform/rd/commondata/cameltest/testsupport/CommonDataFunctionalBaseTest.java
@@ -185,6 +185,30 @@ protected void validateListOfValuesFile(JdbcTemplate jdbcTemplate, String servic
         assertEquals(exceptedResult, listOfValues);
     }
 
+    protected void validateListOfValuesFileWithExternalReference(JdbcTemplate jdbcTemplate, String serviceSql,
+                                            List<Categories> exceptedResult, int size) {
+        RowMapper<Categories> rowMapper = (rs, rowNum) -> {
+            Categories categories = new Categories();
+            categories.setActive(rs.getString("active"));
+            categories.setCategoryKey(rs.getString("categorykey"));
+            categories.setHintTextEN(rs.getString("hinttext_en"));
+            categories.setHintTextCY(rs.getString("hinttext_cy"));
+            categories.setKey(rs.getString("key"));
+            categories.setValueCY(rs.getString("value_cy"));
+            categories.setValueEN(rs.getString("value_en"));
+            categories.setLovOrder(rs.getString("lov_order"));
+            categories.setServiceId(rs.getString("serviceid"));
+            categories.setParentCategory(rs.getString("parentcategory"));
+            categories.setParentKey(rs.getString("parentkey"));
+            categories.setExternalReferenceType(rs.getString("external_reference_type"));
+            categories.setExternalReference(rs.getString("external_reference"));
+            return categories;
+        };
+        var listOfValues = jdbcTemplate.query(serviceSql, rowMapper);
+        assertEquals(size, listOfValues.size());
+        assertEquals(exceptedResult, listOfValues);
+    }
+
     protected void validateFlagServiceFileAudit(JdbcTemplate jdbcTemplate,
                                                 String auditSchedulerQuery, String status, String fileName) {
         var result = jdbcTemplate.queryForList(auditSchedulerQuery);

diff --git a/src/functionalTest/resources/application-functional.yml b/src/functionalTest/resources/application-functional.yml
@@ -84,19 +84,19 @@ route:
     table-name: List_Of_Values
     insert-sql:
       sql:insert into list_of_values (categorykey,serviceid,key,value_en,value_cy,hinttext_en,hinttext_cy,lov_order,
-      parentcategory,parentkey,active)
+      parentcategory,parentkey,active,external_reference,external_reference_type)
       values(:#categoryKey,:#serviceId,:#key,:#value_en,:#value_cy,:#hinttext_en,:#hinttext_cy,:#lov_order,
-      :#parentcategory,:#parentkey,:#active) on conflict (categorykey,key,serviceid) do UPDATE SET categorykey = :#categoryKey,
+      :#parentcategory,:#parentkey,:#active,:#external_reference,:#external_reference_type) on conflict (categorykey,key,serviceid) do UPDATE SET categorykey = :#categoryKey,
       serviceid = :#serviceId, value_en = :#value_en, value_cy = :#value_cy, hinttext_en =:#hinttext_en,
       hinttext_cy = :#hinttext_cy, lov_order = :#lov_order, parentcategory = :#parentcategory, parentkey = :#parentkey,
-      active = :#active?batch=true&dataSource=dataSource
+      active = :#active,external_reference = :#external_reference,external_reference_type = :#external_reference_type?batch=true&dataSource=dataSource
     blob-path:
       azure-blob://${azure.storage.account-name}/rd-common-data/ListOfValues-test.csv?credentials=#credsreg&operation=updateBlockBlob
     processor-class: listOfValuesProcessor
     mapper-class: listOfValuesMapper
     csv-binder-object: ListOfValues
-    csv-headers-expected: categorykey,serviceid,key,value_en,value_cy,hinttext_en,hinttext_cy,lov_order,parentcategory,parentkey,active
-    header-validation-enabled: false
+    csv-headers-expected: categorykey,serviceid,key,value_en,value_cy,hinttext_en,hinttext_cy,lov_order,parentcategory,parentkey,active,external_reference,external_reference_type
+    header-validation-enabled: true
   commondata-case-linking-reasons-load:
     id: commondata-case-linking-reasons-load
     file-name: CaseLinkingReasons-test.csv
@@ -122,18 +122,18 @@ route:
     table-name: List_Of_Values
     insert-sql:
       sql:insert into list_of_values (categorykey,serviceid,key,value_en,value_cy,hinttext_en,hinttext_cy,lov_order,
-      parentcategory,parentkey,active)
+      parentcategory,parentkey,active,external_reference,external_reference_type)
       values(:#categoryKey,:#serviceId,:#key,:#value_en,:#value_cy,:#hinttext_en,:#hinttext_cy,:#lov_order,
-      :#parentcategory,:#parentkey,:#active) on conflict (categorykey,key,serviceid) do UPDATE SET categorykey = :#categoryKey,
+      :#parentcategory,:#parentkey,:#active,:#external_reference,:#external_reference_type) on conflict (categorykey,key,serviceid) do UPDATE SET categorykey = :#categoryKey,
       serviceid = :#serviceId, value_en = :#value_en, value_cy = :#value_cy, hinttext_en =:#hinttext_en,
       hinttext_cy = :#hinttext_cy, lov_order = :#lov_order, parentcategory = :#parentcategory, parentkey = :#parentkey,
-      active = :#active?batch=true&dataSource=dataSource
+      active = :#active,external_reference = :#external_reference,external_reference_type = :#external_reference_type?batch=true&dataSource=dataSource
     blob-path:
       azure-blob://${azure.storage.account-name}/rd-common-data/OtherCategories-test.csv?credentials=#credsreg&operation=updateBlockBlob
     processor-class: listOfValuesProcessor
     mapper-class: listOfValuesMapper
     csv-binder-object: ListOfValues
-    csv-headers-expected: categorykey,serviceid,key,value_en,value_cy,hinttext_en,hinttext_cy,lov_order,parentcategory,parentkey,active
+    csv-headers-expected: categorykey,serviceid,key,value_en,value_cy,hinttext_en,hinttext_cy,lov_order,parentcategory,parentkey,active,external_reference,external_reference_type
     header-validation-enabled: true
 exception-select-query: select * from  dataload_exception_records
 ordered-exception-select-query: select * from  dataload_exception_records order by error_description

diff --git a/src/functionalTest/resources/dbmigration/V1_8__alter_List_Of_Values.sql b/src/functionalTest/resources/dbmigration/V1_8__alter_List_Of_Values.sql
@@ -0,0 +1,8 @@
+
+ALTER TABLE List_Of_Values ADD COLUMN external_reference VARCHAR(200);
+ALTER TABLE List_Of_Values ADD COLUMN external_reference_type VARCHAR(200);
+
+ALTER TABLE List_Of_Values
+ ADD constraint unique_external_reference check
+ ((external_reference_type is null and external_reference is null)
+ or (external_reference_type is not null and external_reference is not null))
diff --git a/src/functionalTest/resources/sourceFiles/categories/list_of_values_0_byte_character.csv b/src/functionalTest/resources/sourceFiles/categories/list_of_values_0_byte_character.csv
@@ -1,4 +1,4 @@
-categorykey,serviceid,key,value_en,value_cy,hinttext_en,hinttext_cy,lov_order,parentcategory,parentkey,active
-AdditionalFacilities,,AF-VF\u200B,Video Facility,,,,,,,Y
-AdditionalFacilities,,AF-SSC,Same Sex Courtroom,,,,,,,Y
-AdditionalFacilities,,AF-WR,Witness  Room,,,,,,,Y
+categorykey,serviceid,key,value_en,value_cy,hinttext_en,hinttext_cy,lov_order,parentcategory,parentkey,active,external_reference,external_reference_type
+AdditionalFacilities,,AF-VF\u200B,Video Facility,,,,,,,Y,,
+AdditionalFacilities,,AF-SSC,Same Sex Courtroom,,,,,,,Y,,
+AdditionalFacilities,,AF-WR,Witness  Room,,,,,,,Y,,
diff --git a/src/functionalTest/resources/sourceFiles/categories/list_of_values_delete_success.csv b/src/functionalTest/resources/sourceFiles/categories/list_of_values_delete_success.csv
@@ -1,4 +1,4 @@
-categorykey,serviceid,key,value_en,value_cy,hinttext_en,hinttext_cy,lov_order,parentcategory,parentkey,active
-AdditionalFacilities,,AF-VF,Video Facility,,,,,,,D
-AdditionalFacilities,,AF-SSC,Same Sex Courtroom,,,,,,,D
-AdditionalFacilities,,AF-WR,Witness Room,,,,,,,D
+categorykey,serviceid,key,value_en,value_cy,hinttext_en,hinttext_cy,lov_order,parentcategory,parentkey,active,external_reference,external_reference_type
+AdditionalFacilities,,AF-VF,Video Facility,,,,,,,D,,
+AdditionalFacilities,,AF-SSC,Same Sex Courtroom,,,,,,,D,,
+AdditionalFacilities,,AF-WR,Witness Room,,,,,,,D,,
diff --git a/src/functionalTest/resources/sourceFiles/categories/list_of_values_duplicate_rec_case1.csv b/src/functionalTest/resources/sourceFiles/categories/list_of_values_duplicate_rec_case1.csv
@@ -1,3 +1,3 @@
-CategoryKey,ServiceID,Key,Value_EN,Value_CY,Hinttext_EN,Hinttext_CY,Lov_Order,ParentCategory,ParentKey,Active
-caseSubType,BBA3,BBA3-001AD,ADVANCE PAYMENT new,,,,,caseType,BBA3-001,Y
-caseSubType,BBA3,BBA3-001AD,ADVANCE PAYMENT scenario1,,,,,caseType,BBA3-001,Y
+CategoryKey,ServiceID,Key,Value_EN,Value_CY,Hinttext_EN,Hinttext_CY,Lov_Order,ParentCategory,ParentKey,Active,external_reference,external_reference_type
+caseSubType,BBA3,BBA3-001AD,ADVANCE PAYMENT new,,,,,caseType,BBA3-001,Y,,
+caseSubType,BBA3,BBA3-001AD,ADVANCE PAYMENT scenario1,,,,,caseType,BBA3-001,Y,,
diff --git a/src/functionalTest/resources/sourceFiles/categories/list_of_values_duplicate_rec_case2.csv b/src/functionalTest/resources/sourceFiles/categories/list_of_values_duplicate_rec_case2.csv
@@ -1,3 +1,3 @@
-CategoryKey,ServiceID,Key,Value_EN,Value_CY,Hinttext_EN,Hinttext_CY,Lov_Order,ParentCategory,ParentKey,Active
-caseSubType,BBA3,BBA3-001AD,ADVANCE PAYMENT new,,,,,caseType,BBA3-001,D
-caseSubType,BBA3,BBA3-001AD,ADVANCE PAYMENT scenario2,,,,,caseType,BBA3-001,Y
+CategoryKey,ServiceID,Key,Value_EN,Value_CY,Hinttext_EN,Hinttext_CY,Lov_Order,ParentCategory,ParentKey,Active,External_Reference,External_Reference_Type
+caseSubType,BBA3,BBA3-001AD,ADVANCE PAYMENT new,,,,,caseType,BBA3-001,D,,
+caseSubType,BBA3,BBA3-001AD,ADVANCE PAYMENT scenario2,,,,,caseType,BBA3-001,Y,,
diff --git a/src/functionalTest/resources/sourceFiles/categories/list_of_values_duplicate_rec_case3.csv b/src/functionalTest/resources/sourceFiles/categories/list_of_values_duplicate_rec_case3.csv
@@ -1,4 +1,4 @@
-CategoryKey,ServiceID,Key,Value_EN,Value_CY,Hinttext_EN,Hinttext_CY,Lov_Order,ParentCategory,ParentKey,Active
-caseSubType,BBA3,BBA3-001AD,ADVANCE PAYMENT new,,,,,caseType,BBA3-001,D
-caseSubType,BBA3,BBA3-001AD,ADVANCE PAYMENT scenario3,,,,,caseType,BBA3-001,Y
-caseSubType,BBA3,BBA3-001AD,ADVANCE PAYMENT scenario3,,,,,caseType,BBA3-001,N
+CategoryKey,ServiceID,Key,Value_EN,Value_CY,Hinttext_EN,Hinttext_CY,Lov_Order,ParentCategory,ParentKey,Active,external_reference,external_reference_type
+caseSubType,BBA3,BBA3-001AD,ADVANCE PAYMENT new,,,,,caseType,BBA3-001,D,,
+caseSubType,BBA3,BBA3-001AD,ADVANCE PAYMENT scenario3,,,,,caseType,BBA3-001,Y,,
+caseSubType,BBA3,BBA3-001AD,ADVANCE PAYMENT scenario3,,,,,caseType,BBA3-001,N,,
diff --git a/src/functionalTest/resources/sourceFiles/categories/list_of_values_duplicate_rec_case4.csv b/src/functionalTest/resources/sourceFiles/categories/list_of_values_duplicate_rec_case4.csv
@@ -1,4 +1,4 @@
-CategoryKey,ServiceID,Key,Value_EN,Value_CY,Hinttext_EN,Hinttext_CY,Lov_Order,ParentCategory,ParentKey,Active
-caseSubType,BBA3,BBA3-001AD,ADVANCE PAYMENT new,,,,,caseType,BBA3-001,D
-caseSubType,BBA3,BBA3-001AD,ADVANCE PAYMENT scenario4,,,,,caseType,BBA3-001,Y
-caseSubType,BBA4,BBA3-001AD,ADVANCE PAYMENT scenario4,,,,,caseType,BBA3-001,N
+CategoryKey,ServiceID,Key,Value_EN,Value_CY,Hinttext_EN,Hinttext_CY,Lov_Order,ParentCategory,ParentKey,Active,external_reference,external_reference_type
+caseSubType,BBA3,BBA3-001AD,ADVANCE PAYMENT new,,,,,caseType,BBA3-001,D,,
+caseSubType,BBA3,BBA3-001AD,ADVANCE PAYMENT scenario4,,,,,caseType,BBA3-001,Y,,
+caseSubType,BBA4,BBA3-001AD,ADVANCE PAYMENT scenario4,,,,,caseType,BBA3-001,N,,