diff --git a/pom.xml b/pom.xml index a939242..cec3d99 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ com.milmove.trdmlambda trdm-lambda - 1.0.1.5 + 1.0.2.0 trdm java spring interface Project for deploying a Java TRDM interfacer for TGET data. diff --git a/src/main/java/com/milmove/trdmlambda/milmove/TrdmCronLambdaHandler.java b/src/main/java/com/milmove/trdmlambda/milmove/TrdmCronLambdaHandler.java index 6065373..1695bbe 100644 --- a/src/main/java/com/milmove/trdmlambda/milmove/TrdmCronLambdaHandler.java +++ b/src/main/java/com/milmove/trdmlambda/milmove/TrdmCronLambdaHandler.java @@ -2,6 +2,7 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.RequestHandler; +import com.milmove.trdmlambda.milmove.handler.LinesOfAccountingHandler; import com.milmove.trdmlambda.milmove.handler.TransportationAccountingCodesHandler; import org.slf4j.LoggerFactory; import org.springframework.boot.SpringApplication; @@ -24,8 +25,13 @@ public String handleRequest(Object input, Context lambdaContext) { logger.info("trdm cron job triggered, starting TAC handler and TGET flow"); TransportationAccountingCodesHandler tacHandler = context.getBean(TransportationAccountingCodesHandler.class); tacHandler.tacCron(); + logger.info("finished tac cron handler"); logger.info("starting LOA handler"); + LinesOfAccountingHandler loaHandler = context.getBean(LinesOfAccountingHandler.class); + loaHandler.loaCron(); + logger.info("finished loa cron handler"); + logger.info("trdm cron job finished execution"); return "trdm cron job finished execution"; } catch (Exception e) { diff --git a/src/main/java/com/milmove/trdmlambda/milmove/handler/LinesOfAccountingHandler.java b/src/main/java/com/milmove/trdmlambda/milmove/handler/LinesOfAccountingHandler.java index d99296d..d66fea7 100644 --- a/src/main/java/com/milmove/trdmlambda/milmove/handler/LinesOfAccountingHandler.java +++ b/src/main/java/com/milmove/trdmlambda/milmove/handler/LinesOfAccountingHandler.java @@ -1,5 +1,50 @@ package com.milmove.trdmlambda.milmove.handler; +import com.milmove.trdmlambda.milmove.util.Trdm; + +import ch.qos.logback.classic.Logger; + +import java.io.IOException; +import java.sql.SQLException; +import java.text.ParseException; + +import javax.xml.datatype.DatatypeConfigurationException; +import javax.xml.datatype.XMLGregorianCalendar; + +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import com.milmove.trdmlambda.milmove.exceptions.TableRequestException; +import com.milmove.trdmlambda.milmove.model.lasttableupdate.LastTableUpdateResponse; + +@Component public class LinesOfAccountingHandler { - // TODO: + private Logger logger = (Logger) LoggerFactory.getLogger(LinesOfAccountingHandler.class); + + private Trdm trdmUtil; + + public LinesOfAccountingHandler(Trdm trdmUtil) { + this.trdmUtil = trdmUtil; + } + + // This cron job will handle the entirety of ensuring the RDS db + // is up to date with proper TGET data. + public void loaCron() + throws SQLException, DatatypeConfigurationException, TableRequestException, IOException, ParseException { + // Gather the last update from TRDM + logger.info("getting lastTableUpdate response with physical name LN_OF_ACCT"); + LastTableUpdateResponse response = trdmUtil.LastTableUpdate("LN_OF_ACCT"); + logger.info("received LastTableUpdateResponse, getting our latest TGET update now"); + XMLGregorianCalendar ourLastUpdate = trdmUtil.GetOurLastTGETUpdate("lines_of_accounting"); + logger.info("received out latest TGET update. Comparing the 2 values to see if our TGET data is out of date"); + boolean tgetOutOfDate = trdmUtil.IsTGETDataOutOfDate(ourLastUpdate, response.getLastUpdate()); + if (tgetOutOfDate) { + logger.info("LOA TGET data is out of date. Starting updateTGETData flow"); + trdmUtil.UpdateTGETData(ourLastUpdate, "LN_OF_ACCT", "lines_of_accounting"); + logger.info("finished updating LOA TGET data"); + } else { + // The data in RDS is up to date, no need to proceed + logger.info("Lines of Accounting RDS Table TGET data already up to date"); + } + } } diff --git a/src/main/java/com/milmove/trdmlambda/milmove/model/LineOfAccounting.java b/src/main/java/com/milmove/trdmlambda/milmove/model/LineOfAccounting.java new file mode 100644 index 0000000..5eb8eb4 --- /dev/null +++ b/src/main/java/com/milmove/trdmlambda/milmove/model/LineOfAccounting.java @@ -0,0 +1,69 @@ +package com.milmove.trdmlambda.milmove.model; + +import lombok.Data; +import java.time.LocalDateTime; +import java.util.UUID; + +@Data +public class LineOfAccounting { + private UUID id; // RDS Internal + private String loaSysID; + private String loaDptID; + private String loaTnsfrDptNm; + private String loaBafID; + private String loaTrsySfxTx; + private String loaMajClmNm; + private String loaOpAgncyID; + private String loaAlltSnID; + private String loaPgmElmntID; + private String loaTskBdgtSblnTx; + private String loaDfAgncyAlctnRcpntID; + private String loaJbOrdNm; + private String loaSbaltmtRcpntID; + private String loaWkCntrRcpntNm; + private String loaMajRmbsmtSrcID; + private String loaDtlRmbsmtSrcID; + private String loaCustNm; + private String loaObjClsID; + private String loaSrvSrcID; + private String loaSpclIntrID; + private String loaBdgtAcntClsNm; + private String loaDocID; + private String loaClsRefID; + private String loaInstlAcntgActID; + private String loaLclInstlID; + private String loaFmsTrnsactnID; + private String loaDscTx; + private LocalDateTime loaBgnDt; + private LocalDateTime loaEndDt; + private String loaFnctPrsNm; + private String loaStatCd; + private String loaHistStatCd; + private String loaHsGdsCd; + private String orgGrpDfasCd; + private String loaUic; + private String loaTrnsnID; + private String loaSubAcntID; + private String loaBetCd; + private String loaFndTyFgCd; + private String loaBgtLnItmID; + private String loaScrtyCoopImplAgncCd; + private String loaScrtyCoopDsgntrCd; + private String loaScrtyCoopLnItmID; + private String loaAgncDsbrCd; + private String loaAgncAcntngCd; + private String loaFndCntrID; + private String loaCstCntrID; + private String loaPrjID; + private String loaActvtyID; + private String loaCstCd; + private String loaWrkOrdID; + private String loaFnclArID; + private String loaScrtyCoopCustCd; + private Integer loaEndFyTx; + private Integer loaBgFyTx; + private String loaBgtRstrCd; + private String loaBgtSubActCd; + private LocalDateTime createdAt; // RDS Internal + private LocalDateTime updatedAt; // RDS Internal +} diff --git a/src/main/java/com/milmove/trdmlambda/milmove/service/DatabaseService.java b/src/main/java/com/milmove/trdmlambda/milmove/service/DatabaseService.java index a395bc0..a87f697 100644 --- a/src/main/java/com/milmove/trdmlambda/milmove/service/DatabaseService.java +++ b/src/main/java/com/milmove/trdmlambda/milmove/service/DatabaseService.java @@ -14,6 +14,7 @@ import java.util.List; import java.util.UUID; +import com.milmove.trdmlambda.milmove.model.LineOfAccounting; import com.milmove.trdmlambda.milmove.model.TransportationAccountingCode; import com.milmove.trdmlambda.milmove.util.SecretFetcher; @@ -61,7 +62,7 @@ public Connection getConnection() throws SQLException { public void insertTransportationAccountingCodes(List codes) throws SQLException { String sql = "INSERT INTO transportation_accounting_codes (id, tac, tac_sys_id, loa_sys_id, tac_fy_txt, tac_fn_bl_mod_cd, org_grp_dfas_cd, tac_mvt_dsg_id, tac_ty_cd, tac_use_cd, tac_maj_clmt_id, tac_bill_act_txt, tac_cost_ctr_nm, buic, tac_hist_cd, tac_stat_cd, trnsprtn_acnt_tx, trnsprtn_acnt_bgn_dt, trnsprtn_acnt_end_dt, dd_actvty_adrs_id, tac_blld_add_frst_ln_tx, tac_blld_add_scnd_ln_tx, tac_blld_add_thrd_ln_tx, tac_blld_add_frth_ln_tx, tac_fnct_poc_nm) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; - + try (Connection conn = this.getConnection(); PreparedStatement pstmt = conn.prepareStatement(sql)) { int count = 0; @@ -105,4 +106,87 @@ public void insertTransportationAccountingCodes(List loas) throws SQLException { + + String sql = "INSERT INTO lines_of_accounting (id, loa_sys_id, loa_dpt_id, loa_tnsfr_dpt_nm, loa_baf_id, loa_trsy_sfx_tx, loa_maj_clm_nm, loa_op_agncy_id, loa_allt_sn_id, loa_pgm_elmnt_id, loa_tsk_bdgt_sbln_tx, loa_df_agncy_alctn_rcpnt_id, loa_jb_ord_nm, loa_sbaltmt_rcpnt_id, loa_wk_cntr_rcpnt_nm, loa_maj_rmbsmt_src_id, loa_dtl_rmbsmt_src_id, loa_cust_nm, loa_obj_cls_id, loa_srv_src_id, loa_spcl_intr_id, loa_bdgt_acnt_cls_nm, loa_doc_id, loa_cls_ref_id, loa_instl_acntg_act_id, loa_lcl_instl_id, loa_fms_trnsactn_id, loa_dsc_tx, loa_bgn_dt, loa_end_dt, loa_fnct_prs_nm, loa_stat_cd, loa_hist_stat_cd, loa_hs_gds_cd, org_grp_dfas_cd, loa_uic, loa_trnsn_id, loa_sub_acnt_id, loa_bet_cd, loa_fnd_ty_fg_cd, loa_bgt_ln_itm_id, loa_scrty_coop_impl_agnc_cd, loa_scrty_coop_dsgntr_cd, loa_scrty_coop_ln_itm_id, loa_agnc_dsbr_cd, loa_agnc_acntng_cd, loa_fnd_cntr_id, loa_cst_cntr_id, loa_prj_id, loa_actvty_id, loa_cst_cd, loa_wrk_ord_id, loa_fncl_ar_id, loa_scrty_coop_cust_cd, loa_end_fy_tx, loa_bg_fy_tx, loa_bgt_rstr_cd, loa_bgt_sub_act_cd, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + + try (Connection conn = this.getConnection(); PreparedStatement pstmt = conn.prepareStatement(sql)) { + + int count = 0; + + for (LineOfAccounting loa : loas) { + if (loa.getId() == null) { + loa.setId(UUID.randomUUID()); + } + pstmt.setObject(1, loa.getId()); + pstmt.setString(2, loa.getLoaSysID()); + pstmt.setString(3, loa.getLoaDptID()); + pstmt.setString(4, loa.getLoaTnsfrDptNm()); + pstmt.setString(5, loa.getLoaBafID()); + pstmt.setString(6, loa.getLoaTrsySfxTx()); + pstmt.setString(7, loa.getLoaMajClmNm()); + pstmt.setString(8, loa.getLoaOpAgncyID()); + pstmt.setString(9, loa.getLoaAlltSnID()); + pstmt.setString(10, loa.getLoaPgmElmntID()); + pstmt.setString(11, loa.getLoaTskBdgtSblnTx()); + pstmt.setString(12, loa.getLoaDfAgncyAlctnRcpntID()); + pstmt.setString(13, loa.getLoaJbOrdNm()); + pstmt.setString(14, loa.getLoaSbaltmtRcpntID()); + pstmt.setString(15, loa.getLoaWkCntrRcpntNm()); + pstmt.setString(16, loa.getLoaMajRmbsmtSrcID()); + pstmt.setString(17, loa.getLoaDtlRmbsmtSrcID()); + pstmt.setString(18, loa.getLoaCustNm()); + pstmt.setString(19, loa.getLoaObjClsID()); + pstmt.setString(20, loa.getLoaSrvSrcID()); + pstmt.setString(21, loa.getLoaSpclIntrID()); + pstmt.setString(22, loa.getLoaBdgtAcntClsNm()); + pstmt.setString(23, loa.getLoaDocID()); + pstmt.setString(24, loa.getLoaClsRefID()); + pstmt.setString(25, loa.getLoaInstlAcntgActID()); + pstmt.setString(26, loa.getLoaLclInstlID()); + pstmt.setString(27, loa.getLoaFmsTrnsactnID()); + pstmt.setString(28, loa.getLoaDscTx()); + pstmt.setDate(29, java.sql.Date.valueOf(loa.getLoaBgnDt().toLocalDate())); + pstmt.setDate(30, java.sql.Date.valueOf(loa.getLoaEndDt().toLocalDate())); + pstmt.setString(31, loa.getLoaFnctPrsNm()); + pstmt.setString(32, loa.getLoaStatCd()); + pstmt.setString(33, loa.getLoaHistStatCd()); + pstmt.setString(34, loa.getLoaHsGdsCd()); + pstmt.setString(35, loa.getOrgGrpDfasCd()); + pstmt.setString(36, loa.getLoaUic()); + pstmt.setString(37, loa.getLoaTrnsnID()); + pstmt.setString(38, loa.getLoaSubAcntID()); + pstmt.setString(39, loa.getLoaBetCd()); + pstmt.setString(40, loa.getLoaFndTyFgCd()); + pstmt.setString(41, loa.getLoaBgtLnItmID()); + pstmt.setString(42, loa.getLoaScrtyCoopImplAgncCd()); + pstmt.setString(43, loa.getLoaScrtyCoopDsgntrCd()); + pstmt.setString(44, loa.getLoaScrtyCoopLnItmID()); + pstmt.setString(45, loa.getLoaAgncDsbrCd()); + pstmt.setString(46, loa.getLoaAgncAcntngCd()); + pstmt.setString(47, loa.getLoaFndCntrID()); + pstmt.setString(48, loa.getLoaCstCntrID()); + pstmt.setString(49, loa.getLoaPrjID()); + pstmt.setString(50, loa.getLoaActvtyID()); + pstmt.setString(51, loa.getLoaCstCd()); + pstmt.setString(52, loa.getLoaWrkOrdID()); + pstmt.setString(53, loa.getLoaFnclArID()); + pstmt.setString(54, loa.getLoaScrtyCoopCustCd()); + pstmt.setObject(55, loa.getLoaEndFyTx()); + pstmt.setObject(56, loa.getLoaBgFyTx()); + pstmt.setString(57, loa.getLoaBgtRstrCd()); + pstmt.setString(58, loa.getLoaBgtSubActCd()); + pstmt.setTimestamp(60, java.sql.Timestamp.valueOf(loa.getUpdatedAt())); + pstmt.addBatch(); + + // Execute every 10000 rows or when finished with the provided LOAs + if (count++ % 10000 == 0 || count == loas.size()) { + pstmt.executeBatch(); + } + } + } + } + } diff --git a/src/main/java/com/milmove/trdmlambda/milmove/util/LineOfAccountingParser.java b/src/main/java/com/milmove/trdmlambda/milmove/util/LineOfAccountingParser.java new file mode 100644 index 0000000..7e22c3d --- /dev/null +++ b/src/main/java/com/milmove/trdmlambda/milmove/util/LineOfAccountingParser.java @@ -0,0 +1,192 @@ +package com.milmove.trdmlambda.milmove.util; + +import java.io.ByteArrayInputStream; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.GregorianCalendar; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Scanner; +import java.util.TimeZone; + +import javax.xml.datatype.XMLGregorianCalendar; + +import ch.qos.logback.classic.Logger; + +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import com.milmove.trdmlambda.milmove.model.LineOfAccounting; + +@Component +public class LineOfAccountingParser { + private Logger logger = (Logger) LoggerFactory.getLogger(LineOfAccountingParser.class); + // Create our map for TGET parsing + Map columnNamesAndLocations = new HashMap<>(); + + private static final String[] expectedColumnNames = { + "LOA_SYS_ID", "LOA_DPT_ID", "LOA_TNSFR_DPT_NM", "LOA_BAF_ID", "LOA_TRSY_SFX_TX", "LOA_MAJ_CLM_NM", + "LOA_OP_AGNCY_ID", "LOA_ALLT_SN_ID", "LOA_PGM_ELMNT_ID", "LOA_TSK_BDGT_SBLN_TX", + "LOA_DF_AGNCY_ALCTN_RCPNT_ID", "LOA_JB_ORD_NM", "LOA_SBALTMT_RCPNT_ID", "LOA_WK_CNTR_RCPNT_NM", + "LOA_MAJ_RMBSMT_SRC_ID", "LOA_DTL_RMBSMT_SRC_ID", "LOA_CUST_NM", "LOA_OBJ_CLS_ID", "LOA_SRV_SRC_ID", + "LOA_SPCL_INTR_ID", "LOA_BDGT_ACNT_CLS_NM", "LOA_DOC_ID", "LOA_CLS_REF_ID", "LOA_INSTL_ACNTG_ACT_ID", + "LOA_LCL_INSTL_ID", "LOA_FMS_TRNSACTN_ID", "LOA_DSC_TX", "LOA_BGN_DT", "LOA_END_DT", "LOA_FNCT_PRS_NM", + "LOA_STAT_CD", "LOA_HIST_STAT_CD", "LOA_HS_GDS_CD", "ORG_GRP_DFAS_CD", "LOA_UIC", "LOA_TRNSN_ID", + "LOA_SUB_ACNT_ID", "LOA_BET_CD", "LOA_FND_TY_FG_CD", "LOA_BGT_LN_ITM_ID", "LOA_SCRTY_COOP_IMPL_AGNC_CD", + "LOA_SCRTY_COOP_DSGNTR_CD", "LOA_SCRTY_COOP_LN_ITM_ID", "LOA_AGNC_DSBR_CD", "LOA_AGNC_ACNTNG_CD", + "LOA_FND_CNTR_ID", "LOA_CST_CNTR_ID", "LOA_PRJ_ID", "LOA_ACTVTY_ID", "LOA_CST_CD", "LOA_WRK_ORD_ID", + "LOA_FNCL_AR_ID", "LOA_SCRTY_COOP_CUST_CD", "LOA_END_FY_TX", "LOA_BG_FY_TX", "LOA_BGT_RSTR_CD", + "LOA_BGT_SUB_ACT_CD", "ROW_STS_CD" }; + + public List parse(byte[] fileContent, XMLGregorianCalendar trdmLastUpdate) + throws RuntimeException { + logger.info("beginning to parse LOA TGET data"); + List codes = new ArrayList<>(); + Scanner scanner = new Scanner(new ByteArrayInputStream(fileContent)); + logger.info("skipping the first line and then gathering headers"); + String[] columnHeaders = scanner.nextLine().split("\\|"); // Skip first line and gather headers immediately + + // TODO: Possibly allow for unexpected column names and proceed with the columns + // we are familiar with. This will be a must for LOA. + if (!Arrays.equals(expectedColumnNames, columnHeaders)) { + String message = String.format("Column headers do not match expected format. Received %s and expected %s", + Arrays.toString(columnHeaders), Arrays.toString(expectedColumnNames)); + throw new RuntimeException(message); + } + + // Map their order for when processing the LOA values properly + for (int i = 0; i < columnHeaders.length; i++) { + columnNamesAndLocations.put(columnHeaders[i], i); + } + + logger.info("headers received and mapped, beginning to process every other line"); + // Loop until the last line in the file is found + while (scanner.hasNextLine()) { + String line = scanner.nextLine(); + // "Unclassified" will always be the last line in the file + if (line.equals("Unclassified")) { + logger.info("finished parsing TGET data file from TRDM"); + break; + } + String[] values = line.split("\\|"); + LineOfAccounting code = processLineIntoLOA(values, columnNamesAndLocations, trdmLastUpdate); + + if (code != null) { + codes.add(code); + } + } + logger.info("finished parsing every single line"); + + scanner.close(); + + return codes; + } + + private LineOfAccounting processLineIntoLOA(String[] values, Map columnHeaders, + XMLGregorianCalendar lastLoaUpdate) + throws RuntimeException { + // Check if LOA is empty or if ROW_STS_CD is "DLT" + if (values[columnHeaders.get("LOA_SYS_ID")].isEmpty() + || "DLT".equals(values[columnHeaders.get("ROW_STS_CD")])) { + return null; // Skip this line + } + + try { + LineOfAccounting loa = new LineOfAccounting(); + loa.setLoaSysID(values[columnHeaders.get("LOA_SYS_ID")]); + loa.setLoaDptID(values[columnHeaders.get("LOA_DPT_ID")]); + loa.setLoaTnsfrDptNm(values[columnHeaders.get("LOA_TNSFR_DPT_NM")]); + loa.setLoaBafID(values[columnHeaders.get("LOA_BAF_ID")]); + loa.setLoaTrsySfxTx(values[columnHeaders.get("LOA_TRSY_SFX_TX")]); + loa.setLoaMajClmNm(values[columnHeaders.get("LOA_MAJ_CLM_NM")]); + loa.setLoaOpAgncyID(values[columnHeaders.get("LOA_OP_AGNCY_ID")]); + loa.setLoaAlltSnID(values[columnHeaders.get("LOA_ALLT_SN_ID")]); + loa.setLoaPgmElmntID(values[columnHeaders.get("LOA_PGM_ELMNT_ID")]); + loa.setLoaTskBdgtSblnTx(values[columnHeaders.get("LOA_TSK_BDGT_SBLN_TX")]); + loa.setLoaDfAgncyAlctnRcpntID(values[columnHeaders.get("LOA_DF_AGNCY_ALCTN_RCPNT_ID")]); + loa.setLoaJbOrdNm(values[columnHeaders.get("LOA_JB_ORD_NM")]); + loa.setLoaSbaltmtRcpntID(values[columnHeaders.get("LOA_SBALTMT_RCPNT_ID")]); + loa.setLoaWkCntrRcpntNm(values[columnHeaders.get("LOA_WK_CNTR_RCPNT_NM")]); + loa.setLoaMajRmbsmtSrcID(values[columnHeaders.get("LOA_MAJ_RMBSMT_SRC_ID")]); + loa.setLoaDtlRmbsmtSrcID(values[columnHeaders.get("LOA_DTL_RMBSMT_SRC_ID")]); + loa.setLoaCustNm(values[columnHeaders.get("LOA_CUST_NM")]); + loa.setLoaObjClsID(values[columnHeaders.get("LOA_OBJ_CLS_ID")]); + loa.setLoaSrvSrcID(values[columnHeaders.get("LOA_SRV_SRC_ID")]); + loa.setLoaSpclIntrID(values[columnHeaders.get("LOA_SPCL_INTR_ID")]); + loa.setLoaBdgtAcntClsNm(values[columnHeaders.get("LOA_BDGT_ACNT_CLS_NM")]); + loa.setLoaDocID(values[columnHeaders.get("LOA_DOC_ID")]); + loa.setLoaClsRefID(values[columnHeaders.get("LOA_CLS_REF_ID")]); + loa.setLoaInstlAcntgActID(values[columnHeaders.get("LOA_INSTL_ACNTG_ACT_ID")]); + loa.setLoaLclInstlID(values[columnHeaders.get("LOA_LCL_INSTL_ID")]); + loa.setLoaFmsTrnsactnID(values[columnHeaders.get("LOA_FMS_TRNSACTN_ID")]); + loa.setLoaDscTx(values[columnHeaders.get("LOA_DSC_TX")]); + if (values[columnHeaders.get("LOA_BGN_DT")] != null) { + LocalDateTime beginDate = LocalDateTime.parse(values[columnHeaders.get("LOA_BGN_DT")], + DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")); + loa.setLoaBgnDt(beginDate); + } + if (values[columnHeaders.get("LOA_END_DT")] != null) { + LocalDateTime endDate = LocalDateTime.parse(values[columnHeaders.get("LOA_END_DT")], + DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")); + loa.setLoaEndDt(endDate); + } + loa.setLoaFnctPrsNm(values[columnHeaders.get("LOA_FNCT_PRS_NM")]); + loa.setLoaStatCd(values[columnHeaders.get("LOA_STAT_CD")]); + loa.setLoaHistStatCd(values[columnHeaders.get("LOA_HIST_STAT_CD")]); + loa.setLoaHsGdsCd(values[columnHeaders.get("LOA_HS_GDS_CD")]); + loa.setOrgGrpDfasCd(values[columnHeaders.get("ORG_GRP_DFAS_CD")]); + loa.setLoaUic(values[columnHeaders.get("LOA_UIC")]); + loa.setLoaTrnsnID(values[columnHeaders.get("LOA_TRNSN_ID")]); + loa.setLoaSubAcntID(values[columnHeaders.get("LOA_SUB_ACNT_ID")]); + loa.setLoaBetCd(values[columnHeaders.get("LOA_BET_CD")]); + loa.setLoaFndTyFgCd(values[columnHeaders.get("LOA_FND_TY_FG_CD")]); + loa.setLoaBgtLnItmID(values[columnHeaders.get("LOA_BGT_LN_ITM_ID")]); + loa.setLoaScrtyCoopImplAgncCd(values[columnHeaders.get("LOA_SCRTY_COOP_IMPL_AGNC_CD")]); + loa.setLoaScrtyCoopDsgntrCd(values[columnHeaders.get("LOA_SCRTY_COOP_DSGNTR_CD")]); + loa.setLoaScrtyCoopLnItmID(values[columnHeaders.get("LOA_SCRTY_COOP_LN_ITM_ID")]); + loa.setLoaAgncDsbrCd(values[columnHeaders.get("LOA_AGNC_DSBR_CD")]); + loa.setLoaAgncAcntngCd(values[columnHeaders.get("LOA_AGNC_ACNTNG_CD")]); + loa.setLoaFndCntrID(values[columnHeaders.get("LOA_FND_CNTR_ID")]); + loa.setLoaCstCntrID(values[columnHeaders.get("LOA_CST_CNTR_ID")]); + loa.setLoaPrjID(values[columnHeaders.get("LOA_PRJ_ID")]); + loa.setLoaActvtyID(values[columnHeaders.get("LOA_ACTVTY_ID")]); + loa.setLoaCstCd(values[columnHeaders.get("LOA_CST_CD")]); + loa.setLoaWrkOrdID(values[columnHeaders.get("LOA_WRK_ORD_ID")]); + loa.setLoaFnclArID(values[columnHeaders.get("LOA_FNCL_AR_ID")]); + loa.setLoaScrtyCoopCustCd(values[columnHeaders.get("LOA_SCRTY_COOP_CUST_CD")]); + String loaEndFyTxValue = values[columnHeaders.get("LOA_END_FY_TX")]; + if (!loaEndFyTxValue.equals("")) { + loa.setLoaEndFyTx(Integer.parseInt(loaEndFyTxValue)); + } else { + loa.setLoaEndFyTx(null); + } + String loaBgFyTxValue = values[columnHeaders.get("LOA_BG_FY_TX")]; + if (!loaBgFyTxValue.equals("")) { + loa.setLoaBgFyTx(Integer.parseInt(loaBgFyTxValue)); + } else { + loa.setLoaBgFyTx(null); + } + loa.setLoaBgtRstrCd(values[columnHeaders.get("LOA_BGT_RSTR_CD")]); + loa.setLoaBgtSubActCd(values[columnHeaders.get("LOA_BGT_SUB_ACT_CD")]); + loa.setUpdatedAt(convertXMLGregorianCalendarToLocalDateTime(lastLoaUpdate)); + return loa; + } catch (DateTimeParseException e) { + throw new RuntimeException("Error parsing dates: " + e.getMessage()); + } + } + + private LocalDateTime convertXMLGregorianCalendarToLocalDateTime(XMLGregorianCalendar xmlGregorianCalendar) { + if (xmlGregorianCalendar == null) { + return null; + } + GregorianCalendar gregorianCalendar = xmlGregorianCalendar.toGregorianCalendar(); + gregorianCalendar.setTimeZone(TimeZone.getTimeZone("UTC")); + return gregorianCalendar.toZonedDateTime().withZoneSameInstant(ZoneId.of("UTC")).toLocalDateTime(); + } + +} \ No newline at end of file diff --git a/src/main/java/com/milmove/trdmlambda/milmove/util/TransportationAccountingCodeParser.java b/src/main/java/com/milmove/trdmlambda/milmove/util/TransportationAccountingCodeParser.java index 548d521..c28683d 100644 --- a/src/main/java/com/milmove/trdmlambda/milmove/util/TransportationAccountingCodeParser.java +++ b/src/main/java/com/milmove/trdmlambda/milmove/util/TransportationAccountingCodeParser.java @@ -2,14 +2,19 @@ import java.io.ByteArrayInputStream; import java.time.LocalDateTime; +import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.util.ArrayList; import java.util.Arrays; +import java.util.GregorianCalendar; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Scanner; +import java.util.TimeZone; + +import javax.xml.datatype.XMLGregorianCalendar; import ch.qos.logback.classic.Logger; @@ -21,6 +26,7 @@ @Component public class TransportationAccountingCodeParser { private Logger logger = (Logger) LoggerFactory.getLogger(TransportationAccountingCodeParser.class); + // Create our map for TGET parsing Map columnNamesAndLocations = new HashMap<>(); @@ -32,12 +38,13 @@ public class TransportationAccountingCodeParser { "TAC_BLLD_ADD_FRTH_LN_TX", "TAC_FNCT_POC_NM", "ROW_STS_CD" }; - public List parse(byte[] fileContent) throws RuntimeException { + public List parse(byte[] fileContent, XMLGregorianCalendar trdmLastUpdate) + throws RuntimeException { logger.info("beginning to parse TAC TGET data"); List codes = new ArrayList<>(); Scanner scanner = new Scanner(new ByteArrayInputStream(fileContent)); logger.info("skipping the first line and then gathering headers"); - String[] columnHeaders = scanner.nextLine().split("\\|"); // Skip first lie and gather headers immediately + String[] columnHeaders = scanner.nextLine().split("\\|"); // Skip first line and gather headers immediately // TODO: Possibly allow for unexpected column names and proceed with the columns // we are familiar with. This will be a must for LOA. @@ -62,7 +69,7 @@ public List parse(byte[] fileContent) throws Runti break; } String[] values = line.split("\\|"); - TransportationAccountingCode code = processLineIntoTAC(values, columnNamesAndLocations); + TransportationAccountingCode code = processLineIntoTAC(values, columnNamesAndLocations, trdmLastUpdate); if (code != null) { codes.add(code); @@ -75,16 +82,20 @@ public List parse(byte[] fileContent) throws Runti return codes; } - private TransportationAccountingCode processLineIntoTAC(String[] values, Map columnHeaders) throws RuntimeException { + private TransportationAccountingCode processLineIntoTAC(String[] values, Map columnHeaders, + XMLGregorianCalendar trdmLastUpdate) throws RuntimeException { // Check if TAC is empty or if ROW_STS_CD is "DLT" - if (values[columnHeaders.get("TRNSPRTN_ACNT_CD")].isEmpty() || "DLT".equals(values[columnHeaders.get("ROW_STS_CD")])) { + if (values[columnHeaders.get("TRNSPRTN_ACNT_CD")].isEmpty() + || "DLT".equals(values[columnHeaders.get("ROW_STS_CD")])) { return null; // Skip this line } - + try { - LocalDateTime effectiveDate = LocalDateTime.parse(values[columnHeaders.get("TRNSPRTN_ACNT_BGN_DT")], DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")); - LocalDateTime expiredDate = LocalDateTime.parse(values[columnHeaders.get("TRNSPRTN_ACNT_END_DT")], DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")); - + LocalDateTime effectiveDate = LocalDateTime.parse(values[columnHeaders.get("TRNSPRTN_ACNT_BGN_DT")], + DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")); + LocalDateTime expiredDate = LocalDateTime.parse(values[columnHeaders.get("TRNSPRTN_ACNT_END_DT")], + DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")); + TransportationAccountingCode code = new TransportationAccountingCode(); code.setTacSysID(values[columnHeaders.get("TAC_SYS_ID")]); code.setLoaSysID(values[columnHeaders.get("LOA_SYS_ID")]); @@ -110,10 +121,20 @@ private TransportationAccountingCode processLineIntoTAC(String[] values, Map allowedTableNames = Set.of("transportation_accounting_codes", "lines_of_accounting"); // RDS @@ -53,12 +58,14 @@ public class Trdm { public Trdm(LastTableUpdateService lastTableUpdateService, GetTableService getTableService, DatabaseService databaseService, - TransportationAccountingCodeParser tacParser) throws SQLException { + TransportationAccountingCodeParser tacParser, + LineOfAccountingParser loaParser) throws SQLException { this.lastTableUpdateService = lastTableUpdateService; this.getTableService = getTableService; this.databaseService = databaseService; this.tacParser = tacParser; + this.loaParser = loaParser; rdsConnection = databaseService.getConnection(); @@ -142,15 +149,24 @@ public void UpdateTGETData(XMLGregorianCalendar ourLastUpdate, String trdmTable, case "transportation_accounting_codes": // Parse the response attachment to get the codes logger.info("parsing response back from TRDM getTable"); - List codes = tacParser.parse(getTableResponse.getAttachment()); + List codes = tacParser.parse(getTableResponse.getAttachment(), + getTableResponse.getDateTime()); logger.info("inserting TACs into DB"); databaseService.insertTransportationAccountingCodes(codes); logger.info("finished inserting TACs into DB"); break; case "lines_of_accounting": - // TODO: + // Parse the response attachment to get the loas + logger.info("parsing response back from TRDM getTable"); + List loas = loaParser.parse(getTableResponse.getAttachment(), + getTableResponse.getDateTime()); + logger.info("inserting LOAs into DB"); + databaseService.insertLinesOfAccounting(loas); + logger.info("finished inserting LOAs into DB"); + break; default: throw new IllegalArgumentException("Invalid rds table name"); } } + } diff --git a/src/test/java/com/milmove/trdmlambda/milmove/util/LineOfAccountingParserTest.java b/src/test/java/com/milmove/trdmlambda/milmove/util/LineOfAccountingParserTest.java new file mode 100644 index 0000000..b891eaf --- /dev/null +++ b/src/test/java/com/milmove/trdmlambda/milmove/util/LineOfAccountingParserTest.java @@ -0,0 +1,108 @@ +package com.milmove.trdmlambda.milmove.util; + +import org.junit.jupiter.api.Test; + +import com.milmove.trdmlambda.milmove.model.LineOfAccounting; + +import static org.junit.jupiter.api.Assertions.*; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.GregorianCalendar; +import java.util.List; +import java.util.TimeZone; + +import javax.xml.datatype.DatatypeConfigurationException; +import javax.xml.datatype.DatatypeFactory; +import javax.xml.datatype.XMLGregorianCalendar; + +class LineOfAccountingParserTest { + + @Test + void testLoaParser() throws IOException, DatatypeConfigurationException { + byte[] bytes = Files.readAllBytes(Paths.get("src/test/resources/Line_Of_Accounting.txt")); + final LineOfAccountingParser lineOfAccountingParser = new LineOfAccountingParser(); + + GregorianCalendar gregorianCalendar = new GregorianCalendar(); + XMLGregorianCalendar today = DatatypeFactory.newInstance().newXMLGregorianCalendar(gregorianCalendar); + + List result = lineOfAccountingParser.parse(bytes, today); + + assertNotNull(result); + assertFalse(result.isEmpty()); + + LineOfAccounting expectedCode = new LineOfAccounting(); + expectedCode.setLoaSysID("124641"); + expectedCode.setLoaDptID("97"); + expectedCode.setLoaTnsfrDptNm(""); + expectedCode.setLoaBafID("4930"); + expectedCode.setLoaTrsySfxTx("AA37"); + expectedCode.setLoaMajClmNm(""); + expectedCode.setLoaOpAgncyID("6D"); + expectedCode.setLoaAlltSnID("0000"); + expectedCode.setLoaPgmElmntID("MZZF0000"); + expectedCode.setLoaTskBdgtSblnTx(""); + expectedCode.setLoaDfAgncyAlctnRcpntID(""); + expectedCode.setLoaJbOrdNm(""); + expectedCode.setLoaSbaltmtRcpntID(""); + expectedCode.setLoaWkCntrRcpntNm("D0000"); + expectedCode.setLoaMajRmbsmtSrcID(""); + expectedCode.setLoaDtlRmbsmtSrcID(""); + expectedCode.setLoaCustNm(""); + expectedCode.setLoaObjClsID("22NL"); + expectedCode.setLoaSrvSrcID(""); + expectedCode.setLoaSpclIntrID(""); + expectedCode.setLoaBdgtAcntClsNm("MA1MN4"); + expectedCode.setLoaDocID("FRT1MNUSAF7790"); + expectedCode.setLoaClsRefID(""); + expectedCode.setLoaInstlAcntgActID("011000"); + expectedCode.setLoaLclInstlID(""); + expectedCode.setLoaFmsTrnsactnID(""); + expectedCode.setLoaDscTx("CONUS ENTRY"); + expectedCode.setLoaBgnDt(LocalDateTime.of(2006, 10, 1, 0, 0)); + expectedCode.setLoaEndDt(LocalDateTime.of(2007, 9, 30, 0, 0)); + expectedCode.setLoaFnctPrsNm(""); + expectedCode.setLoaStatCd("U"); + expectedCode.setLoaHistStatCd(""); + expectedCode.setLoaHsGdsCd(""); + expectedCode.setOrgGrpDfasCd("DZ"); + expectedCode.setLoaUic(""); + expectedCode.setLoaTrnsnID(""); + expectedCode.setLoaSubAcntID(""); + expectedCode.setLoaBetCd(""); + expectedCode.setLoaFndTyFgCd(""); + expectedCode.setLoaBgtLnItmID(""); + expectedCode.setLoaScrtyCoopImplAgncCd(""); + expectedCode.setLoaScrtyCoopDsgntrCd(""); + expectedCode.setLoaScrtyCoopLnItmID(""); + expectedCode.setLoaAgncDsbrCd(""); + expectedCode.setLoaAgncAcntngCd(""); + expectedCode.setLoaFndCntrID(""); + expectedCode.setLoaCstCntrID(""); + expectedCode.setLoaPrjID(""); + expectedCode.setLoaActvtyID(""); + expectedCode.setLoaCstCd(""); + expectedCode.setLoaWrkOrdID(""); + expectedCode.setLoaFnclArID(""); + expectedCode.setLoaScrtyCoopCustCd(""); + expectedCode.setLoaEndFyTx(null); + expectedCode.setLoaBgFyTx(null); + expectedCode.setLoaBgtRstrCd(""); + expectedCode.setLoaBgtSubActCd(""); + expectedCode.setUpdatedAt(convertXMLGregorianCalendarToLocalDateTime(today)); + + assertEquals(expectedCode, result.get(0)); + } + + private LocalDateTime convertXMLGregorianCalendarToLocalDateTime(XMLGregorianCalendar xmlGregorianCalendar) { + if (xmlGregorianCalendar == null) { + return null; + } + GregorianCalendar gregorianCalendar = xmlGregorianCalendar.toGregorianCalendar(); + gregorianCalendar.setTimeZone(TimeZone.getTimeZone("UTC")); + return gregorianCalendar.toZonedDateTime().withZoneSameInstant(ZoneId.of("UTC")).toLocalDateTime(); + } +} diff --git a/src/test/java/com/milmove/trdmlambda/milmove/util/TransportationAccountingCodeParserTest.java b/src/test/java/com/milmove/trdmlambda/milmove/util/TransportationAccountingCodeParserTest.java index 782b69f..15a0041 100644 --- a/src/test/java/com/milmove/trdmlambda/milmove/util/TransportationAccountingCodeParserTest.java +++ b/src/test/java/com/milmove/trdmlambda/milmove/util/TransportationAccountingCodeParserTest.java @@ -10,16 +10,26 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.GregorianCalendar; import java.util.List; +import java.util.TimeZone; + +import javax.xml.datatype.DatatypeConfigurationException; +import javax.xml.datatype.DatatypeFactory; +import javax.xml.datatype.XMLGregorianCalendar; class TransportationAccountingCodeParserTest { @Test - void testTacParser() throws IOException { + void testTacParser() throws IOException, DatatypeConfigurationException { byte[] bytes = Files.readAllBytes(Paths.get("src/test/resources/Transportation_Account.txt")); TransportationAccountingCodeParser parser = new TransportationAccountingCodeParser(); - List result = parser.parse(bytes); + GregorianCalendar gregorianCalendar = new GregorianCalendar(); + XMLGregorianCalendar today = DatatypeFactory.newInstance().newXMLGregorianCalendar(gregorianCalendar); + + List result = parser.parse(bytes, today); assertNotNull(result); assertFalse(result.isEmpty()); @@ -49,7 +59,17 @@ void testTacParser() throws IOException { expectedCode.setTacBillActTxt(""); expectedCode.setBuic(""); expectedCode.setTacHistCd(""); + expectedCode.setUpdatedAt(convertXMLGregorianCalendarToLocalDateTime(today)); assertEquals(expectedCode, result.get(0)); } + + private LocalDateTime convertXMLGregorianCalendarToLocalDateTime(XMLGregorianCalendar xmlGregorianCalendar) { + if (xmlGregorianCalendar == null) { + return null; + } + GregorianCalendar gregorianCalendar = xmlGregorianCalendar.toGregorianCalendar(); + gregorianCalendar.setTimeZone(TimeZone.getTimeZone("UTC")); + return gregorianCalendar.toZonedDateTime().withZoneSameInstant(ZoneId.of("UTC")).toLocalDateTime(); + } } diff --git a/src/test/resources/Line_Of_Accounting.txt b/src/test/resources/Line_Of_Accounting.txt new file mode 100644 index 0000000..db49a50 --- /dev/null +++ b/src/test/resources/Line_Of_Accounting.txt @@ -0,0 +1,10 @@ +LOA_SYS_ID|LOA_DPT_ID|LOA_TNSFR_DPT_NM|LOA_BAF_ID|LOA_TRSY_SFX_TX|LOA_MAJ_CLM_NM|LOA_OP_AGNCY_ID|LOA_ALLT_SN_ID|LOA_PGM_ELMNT_ID|LOA_TSK_BDGT_SBLN_TX|LOA_DF_AGNCY_ALCTN_RCPNT_ID|LOA_JB_ORD_NM|LOA_SBALTMT_RCPNT_ID|LOA_WK_CNTR_RCPNT_NM|LOA_MAJ_RMBSMT_SRC_ID|LOA_DTL_RMBSMT_SRC_ID|LOA_CUST_NM|LOA_OBJ_CLS_ID|LOA_SRV_SRC_ID|LOA_SPCL_INTR_ID|LOA_BDGT_ACNT_CLS_NM|LOA_DOC_ID|LOA_CLS_REF_ID|LOA_INSTL_ACNTG_ACT_ID|LOA_LCL_INSTL_ID|LOA_FMS_TRNSACTN_ID|LOA_DSC_TX|LOA_BGN_DT|LOA_END_DT|LOA_FNCT_PRS_NM|LOA_STAT_CD|LOA_HIST_STAT_CD|LOA_HS_GDS_CD|ORG_GRP_DFAS_CD|LOA_UIC|LOA_TRNSN_ID|LOA_SUB_ACNT_ID|LOA_BET_CD|LOA_FND_TY_FG_CD|LOA_BGT_LN_ITM_ID|LOA_SCRTY_COOP_IMPL_AGNC_CD|LOA_SCRTY_COOP_DSGNTR_CD|LOA_SCRTY_COOP_LN_ITM_ID|LOA_AGNC_DSBR_CD|LOA_AGNC_ACNTNG_CD|LOA_FND_CNTR_ID|LOA_CST_CNTR_ID|LOA_PRJ_ID|LOA_ACTVTY_ID|LOA_CST_CD|LOA_WRK_ORD_ID|LOA_FNCL_AR_ID|LOA_SCRTY_COOP_CUST_CD|LOA_END_FY_TX|LOA_BG_FY_TX|LOA_BGT_RSTR_CD|LOA_BGT_SUB_ACT_CD|ROW_STS_CD +124641|97||4930|AA37||6D|0000|MZZF0000|||||D0000||||22NL|||MA1MN4|FRT1MNUSAF7790||011000|||CONUS ENTRY|2006-10-01 00:00:00|2007-09-30 00:00:00||U|||DZ||||||||||||||||||||||||ACTV +124642|97||4930|AA37||6D|0000|MZZF0000|||||D0000||||22N2|||MA1MN4|FRT1MNUSAF8790||011000|||OCONUS ENTRY|2006-10-01 00:00:00|2007-09-30 00:00:00||U|||DZ||||||||||||||||||||||||ACTV +124643|97||4930|AA37||6D|0000|MZZF0000|||||D0000||||22NL|||P50MD6|FRT0MDUSAF9790||011000|||ENTRY TYPE A|2006-10-01 00:00:00|2007-09-30 00:00:00||U|||DZ||||||||||||||||||||||||ACTV +124644|97||4930|AA37||6D|0000|MZZF0000|||||D0000||||22NL|||P50MN6|FRT0MNUSAF1790||011000|||ADDITIONAL ENTRY|2006-10-01 00:00:00|2007-09-30 00:00:00||U|||DZ||||||||||||||||||||||||ACTV +124645|97||4930|AA37||6D|0000|MZZF0000|||||D0000||||22NL|||P51MD6|FRT1MDUSAF2790||011000|||FURNITURE KIT|2006-10-01 00:00:00|2007-09-30 00:00:00||U|||DZ||||||||||||||||||||||||ACTV +124646|97||4930|AA37||6D|0000|MZZF0000|||||D0000||||22NL|||P51MN6|FRT1MNUSAF3790||011000|||PIECE FOR REPAIR|2006-10-01 00:00:00|2007-09-30 00:00:00||U|||DZ||||||||||||||||||||||||ACTV +124647|97||4930|AA37||6D|0000|MZZF0000|||||D0000||||22NL|||P52MD6|FRT2MDUSAF4790||011000|||ENTRY TYPE B|2006-10-01 00:00:00|2007-09-30 00:00:00||U|||DZ||||||||||||||||||||||||ACTV +124840|97||4930|AC6E||62|AC6E|SM0Z1400|||||||||22N2|||AC6E00|FRTAC6EAAJA790||028043|||PHONE REPAIR KIT|2006-10-01 00:00:00|2007-09-30 00:00:00||U|||DZ||||||||||||||||||||||||ACTV +Unclassified