merging all done tests passed

Dependencies.toml

@@ -5,7 +5,7 @@
 
 [ballerina]
 dependencies-toml-version = "2"
-distribution-version = "2201.10.0"
+distribution-version = "2201.10.1"
 
 [[package]]
 org = "ballerina"
@@ -179,7 +179,7 @@ modules = [
 [[package]]
 org = "ballerinax"
 name = "h2.driver"
-version = "1.0.0"
+version = "1.1.0"
 modules = [
 	{org = "ballerinax", packageName = "h2.driver", moduleName = "h2.driver"}
 ]
@@ -201,7 +201,7 @@ modules = [
 
 [[package]]
 org = "lakshanweerasinghe"
-name = "batch_etl_pipeline"
+name = "batch_etl_pipeline_demo"
 version = "0.1.0"
 dependencies = [
 	{org = "ballerina", name = "io"},
@@ -214,6 +214,6 @@ dependencies = [
 	{org = "ballerinax", name = "java.jdbc"}
 ]
 modules = [
-	{org = "lakshanweerasinghe", packageName = "batch_etl_pipeline", moduleName = "batch_etl_pipeline"}
+	{org = "lakshanweerasinghe", packageName = "batch_etl_pipeline_demo", moduleName = "batch_etl_pipeline_demo"}
 ]
 

main.bal

@@ -3,6 +3,7 @@ import ballerina/sql;
 import ballerina/time;
 import ballerinax/h2.driver as _;
 import ballerinax/java.jdbc;
+import ballerina/io;
 
 final jdbc:Client dbClient = check new (url = "jdbc:h2:file:./database/loandatabase", user = "test", password = "test");
 
@@ -15,74 +16,88 @@ public function main() returns error? {
 
 function extract() returns [LoanRequest[], LoanApproval[]]|error {
     log:printInfo("BEGIN: extract data from the sftp server");
-    // Hint: Use io ballerina library and read the csv files
 
-    string loanRequestFile = "loan_request_2024_03_22.csv";
-    LoanRequest[] loanRequests;
+    string loanRequestFile = "resources/loan_request_2024_03_22.csv";
+    string loanApprovalsFile = "resources/approved_loans_2024_03_22.csv";
 
-    string loanApprovalsFile = "approved_loans_2024_03_22.csv";    
-    LoanApproval[] loanApprovals;
+    // Reads the previously-saved CSV file as a record[].
+    LoanRequest[] readLoanRequests = check io:fileReadCsv(loanRequestFile);
+    LoanApproval[] readLoanApprovals = check io:fileReadCsv(loanApprovalsFile);
+    io:println(readLoanRequests);
+    io:println(readLoanApprovals);
 
     log:printInfo("END: extract data from the sftp server");
-    return [loanRequests, loanApprovals];
+    return [readLoanRequests, readLoanApprovals];
 }
 
 function transform(LoanRequest[] loanRequests, LoanApproval[] loanApprovals)
     returns [Loan[], BranchPerformance[], RegionPerformance[]] {
     log:printInfo("START: transform data");
 
-    // Get the unique approved loan requests by joining two csv files
-    // Create an array of Loan records
-    // Hint: User ballerina integrated queries and transformLoanRequest function
-    Loan[] approvedLoans;
+    // Join LoanRequest and LoanApproval arrays to get the approved loans
+    Loan[] approvedLoans = from LoanRequest lr in loanRequests
+                           join LoanApproval la in loanApprovals
+                           on lr.loanRequestId equals la.loanRequestId
+                           select transformLoanRequest(lr, la);
 
+    // Group the approvedLoans by branch and loan type to create BranchPerformance records
     BranchPerformance[] branchPerformance = from var {branch, loanType, grantedAmount, interest}
-        in approvedLoans
-        group by branch, loanType
-        select {
-            id: generateId(),
-            branch,
-            loanType,
-            totalGrants: sum(grantedAmount),
-            totalInterest: sum(interest),
-            date: todayString()
-        };
-
-    // Group the `approvedLoans` by region, loanType, date, dayOfWeek
-    // Hint: User ballerina integrated queries and use `sum` function when needed
-    RegionPerformance[] regionPerformance;
+                                            in approvedLoans
+                                            group by branch, loanType
+                                            select {
+                                                id: generateId(),
+                                                branch,
+                                                loanType,
+                                                totalGrants: sum(grantedAmount),
+                                                totalInterest: sum(interest),
+                                                date: todayString()
+                                            };
+
+    // Group the approvedLoans by region, loanType, date, dayOfWeek to create RegionPerformance records
+    RegionPerformance[] regionPerformance = from var {region, loanType, date, dayOfWeek, grantedAmount, interest}
+                                            in approvedLoans
+                                            group by region, loanType, date, dayOfWeek
+                                            select {
+                                                id: generateId(),
+                                                region,
+                                                loanType,
+                                                date,
+                                                dayOfWeek,
+                                                totalGrants: sum(grantedAmount),
+                                                totalInterest: sum(interest)
+                                            };
 
     log:printInfo("END: transform data");
     return [approvedLoans, branchPerformance, regionPerformance];
 }
 
 function transformLoanRequest(LoanRequest loanRequest, LoanApproval loanApproval) returns Loan {
-    log:printInfo(string `START: transform loan request: ${loanRequest.loanRequestId}`);
+    //log:printInfo(string START: transform loan request: ${loanRequest.loanRequestId});
 
     var {loanRequestId, amount, loanType, datetime, period, branch, status} = loanRequest;
     var {grantedAmount, interest, period: approvedPeriod} = loanApproval;
 
-    // date time related operations
+    // Date time related operations
     time:Date date = fromUtcStringToDate(datetime, USA_UTC_OFFSET_IN_SECONDS);
     string dateString = fromDateToString(date);
     DayOfWeek dayOfWeek = getDayOfWeek(date);
 
-    // Hint: Categorize branch by region
-    string region;
+    // Categorize branch by region
+    string region = getRegion(branch);
 
-    // Hint: Catergorization of loans by amount and type
-    LoanCatergotyByAmount loanCatergoryByAmount;
+    // Categorization of loans by amount and type
+    LoanCatergotyByAmount loanCatergoryByAmount = getLoanCategoryByAmount(amount, loanType);
 
-    // Hint: Calculate total interest
-    decimal totalInterest;
+    // Calculate total interest
+    decimal totalInterest = interest;
 
-    // Hint: Get the loan status
-    LoanStatus loanStatus;
+    // Get the loan status
+    LoanStatus loanStatus = getLoanStatus(status);
 
-    // Hint: Get the loan type
-    LoanType 'type;
+    // Get the loan type
+    LoanType 'type = getLoanType(loanType);
 
-    log:printInfo(string `END: transform loan request: ${loanRequest.loanRequestId}`);
+    //log:printInfo(string END: transform loan request: ${loanRequest.loanRequestId});
     return {
         loanRequestId,
         amount,
@@ -115,7 +130,7 @@ function loadRegionPerformance(RegionPerformance[] data) returns error? {
                 (id, region, loanType, date, dayOfWeek, totalGrants, totalInterest) 
                 VALUES (${rp.id}, ${rp.region}, ${rp.loanType}, 
                 ${rp.date}, ${rp.dayOfWeek}, ${rp.totalGrants}, ${rp.totalInterest})`;
-   _ = check dbClient->batchExecute(insertQueries);
+    _ = check dbClient->batchExecute(insertQueries);
 }
 
 function loadBranchPerformance(BranchPerformance[] data) returns error? {
@@ -135,4 +150,4 @@ function loadLoan(Loan[] data) returns error? {
 
 function getRegion(string branch) returns string {
     return branchToRegionMap[branch] ?: "";
-}
+}

target/build

@@ -0,0 +1,8 @@
+{
+  "last_build_time": 1728910519646,
+  "last_update_time": 1728898200018,
+  "distribution_version": "2201.10.1",
+  "last_modified_time": {
+    "batch_etl_pipeline_demo": 1728898158874
+  }
+}

target/cache/tests_cache/native-config/mocked-func-class-map.json

@@ -0,0 +1 @@
+{"batch_etl_pipeline_demo/tests.test":[]}

target/cache/tests_cache/test_suit.json

@@ -0,0 +1 @@
+{"batch_etl_pipeline_demo":{"orgName":"lakshanweerasinghe","version":"0.1.0","packageName":"batch_etl_pipeline_demo","packageId":"batch_etl_pipeline_demo","testPackageId":"batch_etl_pipeline_demo$test","executeFilePath":"tests.test_execute-generated_1","sourceRootPath":"C:\\Users\\Tharuka Sandaruwan\\Desktop\\Ballerina sesssion\\batch-etl-pipeline-demo","testUtilityFunctions":{"todayString":"lakshanweerasinghe.batch_etl_pipeline_demo.0.utils","getLoanStatus":"lakshanweerasinghe.batch_etl_pipeline_demo.0.utils","initDB":"lakshanweerasinghe.batch_etl_pipeline_demo.0.init_db","fromUtcStringToDate":"lakshanweerasinghe.batch_etl_pipeline_demo.0.utils","testLoan":"lakshanweerasinghe.batch_etl_pipeline_demo$test.0.tests.test","executeTestRegistrar0":"lakshanweerasinghe.batch_etl_pipeline_demo$test.0.tests.test_execute-generated_1","generateId":"lakshanweerasinghe.batch_etl_pipeline_demo.0.utils","loadLoan":"lakshanweerasinghe.batch_etl_pipeline_demo.0.main","getLoanType":"lakshanweerasinghe.batch_etl_pipeline_demo.0.utils","main":"lakshanweerasinghe.batch_etl_pipeline_demo.0.main","__execute__":"lakshanweerasinghe.batch_etl_pipeline_demo$test.0.tests.test_execute-generated_1","transform":"lakshanweerasinghe.batch_etl_pipeline_demo.0.main","getDayOfWeek":"lakshanweerasinghe.batch_etl_pipeline_demo.0.utils","fromDateToString":"lakshanweerasinghe.batch_etl_pipeline_demo.0.utils","loadRegionPerformance":"lakshanweerasinghe.batch_etl_pipeline_demo.0.main","extract":"lakshanweerasinghe.batch_etl_pipeline_demo.0.main","load":"lakshanweerasinghe.batch_etl_pipeline_demo.0.main","transformLoanRequest":"lakshanweerasinghe.batch_etl_pipeline_demo.0.main","getLoanCategoryByAmount":"lakshanweerasinghe.batch_etl_pipeline_demo.0.utils","getRegion":"lakshanweerasinghe.batch_etl_pipeline_demo.0.main","testRegionPerformance":"lakshanweerasinghe.batch_etl_pipeline_demo$test.0.tests.test","loadBranchPerformance":"lakshanweerasinghe.batch_etl_pipeline_demo.0.main","testBranchPerformance":"lakshanweerasinghe.batch_etl_pipeline_demo$test.0.tests.test","runETL":"lakshanweerasinghe.batch_etl_pipeline_demo$test.0.tests.test"},"beforeSuiteFunctionNames":[],"afterSuiteFunctionNames":{},"beforeEachFunctionNames":[],"afterEachFunctionNames":[],"tests":[],"groups":{},"testExecutionDependencies":["C:\\PROGRA~1\\Ballerina\\distributions\\ballerina-2201.10.1\\bin\\..\\repo\\bala\\ballerina\\time\\2.5.0\\java17\\platform\\java17\\time-native-2.5.0.jar","C:\\Users\\Tharuka Sandaruwan\\.ballerina\\repositories\\central.ballerina.io\\bala\\ballerinax\\java.jdbc\\1.12.1\\java17\\platform\\java17\\java.jdbc-native-1.12.1.jar","C:\\PROGRA~1\\Ballerina\\distributions\\ballerina-2201.10.1\\bin\\..\\repo\\bala\\ballerina\\io\\1.6.1\\java17\\platform\\java17\\io-native-1.6.1.jar","C:\\PROGRA~1\\Ballerina\\distributions\\ballerina-2201.10.1\\bin\\..\\repo\\bala\\ballerina\\test\\0.0.0\\any\\platform\\java17\\testerina-core-2201.10.1.jar","C:\\PROGRA~1\\Ballerina\\distributions\\ballerina-2201.10.1\\bin\\..\\repo\\cache\\ballerina\\observe\\1.3.0\\java17\\ballerina-observe-1.3.0.jar","C:\\PROGRA~1\\Ballerina\\distributions\\ballerina-2201.10.1\\bin\\..\\repo\\cache\\ballerina\\crypto\\2.7.2\\java17\\ballerina-crypto-2.7.2.jar","C:\\Users\\Tharuka Sandaruwan\\Desktop\\Ballerina sesssion\\batch-etl-pipeline-demo\\target\\cache\\lakshanweerasinghe\\batch_etl_pipeline_demo\\0.1.0\\java17\\lakshanweerasinghe-batch_etl_pipeline_demo-0.1.0.jar","C:\\PROGRA~1\\Ballerina\\distributions\\ballerina-2201.10.1\\bin\\..\\bre\\lib\\asm-tree-9.7.jar","C:\\Users\\Tharuka Sandaruwan\\.ballerina\\repositories\\central.ballerina.io\\cache-2201.10.1\\ballerina\\sql\\1.14.1\\java17\\ballerina-sql-1.14.1.jar","C:\\PROGRA~1\\Ballerina\\distributions\\ballerina-2201.10.1\\bin\\..\\repo\\cache\\ballerina\\uuid\\1.8.0\\java17\\ballerina-uuid-1.8.0.jar","C:\\Users\\Tharuka Sandaruwan\\.ballerina\\repositories\\central.ballerina.io\\cache-2201.10.1\\ballerinax\\java.jdbc\\1.12.1\\java17\\ballerinax-java.jdbc-1.12.1.jar","C:\\Users\\Tharuka Sandaruwan\\Desktop\\Ballerina sesssion\\batch-etl-pipeline-demo\\target\\cache\\lakshanweerasinghe\\batch_etl_pipeline_demo\\0.1.0\\java17\\resources.jar","C:\\PROGRA~1\\Ballerina\\distributions\\ballerina-2201.10.1\\bin\\..\\bre\\lib\\org.jacoco.core-0.8.12.jar","C:\\PROGRA~1\\Ballerina\\distributions\\ballerina-2201.10.1\\bin\\..\\repo\\cache\\ballerina\\time\\2.5.0\\java17\\ballerina-time-2.5.0.jar","C:\\PROGRA~1\\Ballerina\\distributions\\ballerina-2201.10.1\\bin\\..\\bre\\lib\\asm-9.7.jar","C:\\PROGRA~1\\Ballerina\\distributions\\ballerina-2201.10.1\\bin\\..\\bre\\lib\\org.jacoco.report-0.8.12.jar","C:\\PROGRA~1\\Ballerina\\distributions\\ballerina-2201.10.1\\bin\\..\\repo\\cache\\ballerina\\io\\1.6.1\\java17\\ballerina-io-1.6.1.jar","C:\\Users\\Tharuka Sandaruwan\\Desktop\\Ballerina sesssion\\batch-etl-pipeline-demo\\target\\cache\\lakshanweerasinghe\\batch_etl_pipeline_demo\\0.1.0\\java17\\lakshanweerasinghe-batch_etl_pipeline_demo-0.1.0-testable.jar","C:\\PROGRA~1\\Ballerina\\distributions\\ballerina-2201.10.1\\bin\\..\\repo\\cache\\ballerina\\log\\2.10.0\\java17\\ballerina-log-2.10.0.jar","C:\\Users\\Tharuka Sandaruwan\\.ballerina\\repositories\\central.ballerina.io\\bala\\ballerina\\sql\\1.14.1\\java17\\platform\\java17\\HikariCP-3.3.1.jar","C:\\PROGRA~1\\Ballerina\\distributions\\ballerina-2201.10.1\\bin\\..\\repo\\bala\\ballerina\\observe\\1.3.0\\java17\\platform\\java17\\observe-native-1.3.0.jar","C:\\PROGRA~1\\Ballerina\\distributions\\ballerina-2201.10.1\\bin\\..\\repo\\cache\\ballerina\\observe\\1.3.0\\java17\\ballerina-observe.mockextension-1.3.0.jar","C:\\Users\\Tharuka Sandaruwan\\.ballerina\\repositories\\central.ballerina.io\\bala\\ballerinax\\h2.driver\\1.1.0\\java17\\platform\\java17\\h2-2.2.220.jar","C:\\PROGRA~1\\Ballerina\\distributions\\ballerina-2201.10.1\\bin\\..\\bre\\lib\\ballerina-lang-2201.10.1.jar","C:\\PROGRA~1\\Ballerina\\distributions\\ballerina-2201.10.1\\bin\\..\\repo\\cache\\ballerina\\test\\0.0.0\\java17\\ballerina-test-0.0.0.jar","C:\\PROGRA~1\\Ballerina\\distributions\\ballerina-2201.10.1\\bin\\..\\bre\\lib\\testerina-runtime-2201.10.1.jar","C:\\PROGRA~1\\Ballerina\\distributions\\ballerina-2201.10.1\\bin\\..\\bre\\lib\\asm-commons-9.7.jar","C:\\PROGRA~1\\Ballerina\\distributions\\ballerina-2201.10.1\\bin\\..\\bre\\lib\\java-diff-utils-4.5.jar","C:\\Users\\Tharuka Sandaruwan\\.ballerina\\repositories\\central.ballerina.io\\cache-2201.10.1\\ballerinax\\h2.driver\\1.1.0\\java17\\ballerinax-h2.driver-1.1.0.jar","C:\\PROGRA~1\\Ballerina\\distributions\\ballerina-2201.10.1\\bin\\..\\bre\\lib\\ballerina-rt-2201.10.1.jar","C:\\PROGRA~1\\Ballerina\\distributions\\ballerina-2201.10.1\\bin\\..\\bre\\lib\\testerina-core-2201.10.1.jar","C:\\PROGRA~1\\Ballerina\\distributions\\ballerina-2201.10.1\\bin\\..\\repo\\bala\\ballerina\\crypto\\2.7.2\\java17\\platform\\java17\\crypto-native-2.7.2.jar","C:\\PROGRA~1\\Ballerina\\distributions\\ballerina-2201.10.1\\bin\\..\\repo\\bala\\ballerina\\test\\0.0.0\\any\\platform\\java17\\java-diff-utils-4.5.jar","C:\\PROGRA~1\\Ballerina\\distributions\\ballerina-2201.10.1\\bin\\..\\repo\\bala\\ballerina\\crypto\\2.7.2\\java17\\platform\\java17\\bcpg-jdk18on-1.78.jar","C:\\PROGRA~1\\Ballerina\\distributions\\ballerina-2201.10.1\\bin\\..\\repo\\bala\\ballerina\\observe\\1.3.0\\java17\\platform\\java17\\opentelemetry-sdk-trace-1.0.0.jar","C:\\PROGRA~1\\Ballerina\\distributions\\ballerina-2201.10.1\\bin\\..\\repo\\bala\\ballerina\\log\\2.10.0\\java17\\platform\\java17\\log-native-2.10.0.jar","C:\\PROGRA~1\\Ballerina\\distributions\\ballerina-2201.10.1\\bin\\..\\repo\\bala\\ballerina\\observe\\1.3.0\\java17\\platform\\java17\\opentelemetry-semconv-1.0.0-alpha.jar","C:\\PROGRA~1\\Ballerina\\distributions\\ballerina-2201.10.1\\bin\\..\\repo\\bala\\ballerina\\crypto\\2.7.2\\java17\\platform\\java17\\bcutil-jdk18on-1.78.jar","C:\\PROGRA~1\\Ballerina\\distributions\\ballerina-2201.10.1\\bin\\..\\repo\\bala\\ballerina\\crypto\\2.7.2\\java17\\platform\\java17\\bcpkix-jdk18on-1.78.jar","C:\\PROGRA~1\\Ballerina\\distributions\\ballerina-2201.10.1\\bin\\..\\repo\\bala\\ballerina\\observe\\1.3.0\\java17\\platform\\java17\\opentelemetry-sdk-common-1.0.0.jar","C:\\PROGRA~1\\Ballerina\\distributions\\ballerina-2201.10.1\\bin\\..\\repo\\bala\\ballerina\\observe\\1.3.0\\java17\\platform\\java17\\opentelemetry-sdk-testing-1.0.0.jar","C:\\Users\\Tharuka Sandaruwan\\.ballerina\\repositories\\central.ballerina.io\\bala\\ballerina\\sql\\1.14.1\\java17\\platform\\java17\\sql-native-1.14.1.jar","C:\\PROGRA~1\\Ballerina\\distributions\\ballerina-2201.10.1\\bin\\..\\repo\\bala\\ballerina\\crypto\\2.7.2\\java17\\platform\\java17\\bcprov-jdk18on-1.78.jar","C:\\Users\\Tharuka Sandaruwan\\.ballerina\\repositories\\central.ballerina.io\\bala\\ballerinax\\h2.driver\\1.1.0\\java17\\platform\\java17\\h2.driver-native-1.1.0.jar"],"isReportRequired":false,"isSingleDDTExecution":false,"mockFunctionNamesMap":{}}}

target/rerun_test.json

@@ -0,0 +1 @@
+{"batch_etl_pipeline_demo":{"testNames":[],"testModuleNames":{},"subTestNames":{}}}