package com.inscloudtech.bankStatementAnalysis.helper; import cn.hutool.core.date.DateUtil; import cn.hutool.core.io.FileUtil; import cn.hutool.core.util.StrUtil; import cn.hutool.json.JSON; import cn.hutool.json.JSONObject; import com.alibaba.excel.EasyExcel; import com.alibaba.excel.ExcelReader; import com.alibaba.excel.read.metadata.ReadSheet; import com.alibaba.excel.util.ListUtils; import com.alibaba.excel.util.StringUtils; import com.aspose.cells.Cell; import com.aspose.cells.Cells; import com.aspose.cells.Workbook; import com.aspose.cells.Worksheet; import com.inscloudtech.bankStatementAnalysis.domain.HeadField; import com.inscloudtech.bankStatementAnalysis.helper.HelperUtil; import com.inscloudtech.bankStatementAnalysis.service.ImportService; import com.inscloudtech.common.constant.BankStatementConstants; import com.inscloudtech.common.constant.Constants; import com.inscloudtech.common.exception.dc.AnalyzeDataFailedException; import com.inscloudtech.common.exception.dc.ImportDataFailedException; import com.inscloudtech.common.exception.dc.TemplateNotFindException; import com.inscloudtech.common.utils.bean.BeanUtils; import com.inscloudtech.common.utils.file.FileUtils; import com.inscloudtech.datacenter.domain.PlateNumberInfo; import com.inscloudtech.bankStatementAnalysis.mapper.EsICBCBankStatementMapper; import com.inscloudtech.bankStatementAnalysis.mapper.EsICBCCreditCardBankStatementMapper; import com.inscloudtech.datacenter.service.ImportResultService; import com.inscloudtech.datacenter.domain.BankStatement; import com.inscloudtech.datacenter.domain.OpeningAccountInfo; import com.inscloudtech.bankStatementAnalysis.domain.entity.ICBCBankStatementEntry; import com.inscloudtech.bankStatementAnalysis.domain.entity.ICBCCreditCardBankStatementEntry; import com.inscloudtech.bankStatementAnalysis.util.AsposeUtil; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import java.io.File; import java.math.BigDecimal; import java.util.*; /** * 工商银行数据分析 */ @Slf4j @RequiredArgsConstructor @Component public class ICBCDataAnalysisHelper { private final ImportService importService; private final ImportResultService importResultService; private final EsICBCBankStatementMapper bankStatementMapper; private final EsICBCCreditCardBankStatementMapper creditCardBankStatementMapper; private final static String BANK_NAME = "工商银行"; public void importData(File dir,String caseId) throws Exception { List fileList = FileUtil.loopFiles(dir); if (fileList == null) { return; } List excelFileList = HelperUtil.getExcelFile(fileList); // List pdfFileList = fileList.stream() // .filter(file -> FileUtil.extName(file).equalsIgnoreCase("pdf")) // .collect(Collectors.toList()); // Map> groupByPath = excelFileList.stream().collect(Collectors.groupingBy(File::getAbsolutePath)); // for (File file : pdfFileList) { // String excelFilename = AsposeUtil.pdf2ExcelV6(file.getAbsolutePath()); // if(groupByPath.containsKey(excelFilename)){ // continue; // } // excelFileList.add(new File(excelFilename)); // } for (File f : excelFileList) { try { String absolutePath = f.getAbsolutePath(); Workbook wb = new Workbook(absolutePath); int count = wb.getWorksheets().getCount(); String sourceFile = HelperUtil.getSourceFileName(absolutePath, BANK_NAME); for (int sheetNo = 0; sheetNo < count; sheetNo++) { try { Worksheet worksheet = wb.getWorksheets().get(sheetNo); Cells cells = worksheet.getCells(); Cell contentCell = cells.get(0, 0); if (contentCell.getValue() == null) {//空白excel continue; } String worksheetName = worksheet.getName(); Cell cell420240418 = AsposeUtil.getCell(cells, "入帐日期"); Cell khCell = AsposeUtil.getCell(cells, "开户日期"); Cell tradeCell = AsposeUtil.getCell(cells, "交易时间戳"); Cell khhzh = AsposeUtil.getCell(cells, "卡号或帐号"); String withSheetName = sourceFile + BankStatementConstants.NAME_WITH_SHEET_NAME + worksheetName; /**20240418新增模板*/ if (cell420240418 != null) { readBSFor20240418(cells,absolutePath, sheetNo,withSheetName); }else if (khCell != null) { // 判断是否是开户信息 int rowNumber = khCell.getRow() + 1; importService.readOAIData(absolutePath, rowNumber, sheetNo, caseId, BANK_NAME, withSheetName); } else if (tradeCell != null) { // 流水信息 readBS(absolutePath, sheetNo,withSheetName); } else if (khhzh != null) { // 开户 importService.readOAIData(absolutePath, khhzh.getRow() + 1, sheetNo, caseId, BANK_NAME, withSheetName); } else { throw new TemplateNotFindException(withSheetName); } } catch (Exception e) { importResultService.record(caseId, BANK_NAME, e); } } } catch (Exception e) { importResultService.record(caseId, BANK_NAME, e); } } } private void readBSFor20240418(Cells cells ,String absolutePath, int sheetNo,String sourceFile) { // int startRow = dateCell.getRow(); // Cell cardHolderNameCell = cells.get(startRow - 1, 1); // if(null == cardHolderNameCell || StrUtil.isEmpty(cardHolderNameCell.getStringValue())){ // throw new ImportDataFailedException("读取持卡人信息异常", sourceFile); // } List maps = new LinkedList<>(); int maxRow = cells.getMaxRow(); List headFields = new ArrayList<>(); headFields.add(new HeadField("transactionAmount","发生额")); headFields.add(new HeadField("balance","余额")); headFields.add(new HeadField("accountNumber","商密二级,帐号")); for(int i = 0; i < maxRow; i++){ Cell dateCell = cells.get(i, 2); String stringValue = dateCell.getStringValue(); if (StrUtil.isEmpty(stringValue)) { continue; } if(stringValue.equals("入帐日期")){ int startRow = dateCell.getRow(); Cell cardHolderNameCell = cells.get(startRow - 1, 1); if(null == cardHolderNameCell || StrUtil.isEmpty(cardHolderNameCell.getStringValue())){ throw new ImportDataFailedException("读取持卡人信息异常", sourceFile); } String cardHolderName = cardHolderNameCell.getStringValue().trim(); JSONObject jsonObject = new JSONObject(); jsonObject.putOnce("startRow",i); jsonObject.putOnce("cardHolderName",cardHolderName); maps.add(jsonObject); } } int size = maps.size(); for(int i = 0; i < size; i++){ JSONObject jsonObject = maps.get(i); String cardHolderName = jsonObject.getStr("cardHolderName"); int headRowNumber = jsonObject.getInt("startRow") + 1; int endRow = maxRow -1; if(i + 1 < size){ JSONObject nextRow = maps.get(i + 1); endRow = nextRow.getInt("startRow") - 1; } if(endRow - headRowNumber <= 1){ continue; } importService.readMultiplePersonAndMultipleHeadBankStatement(absolutePath,headRowNumber,endRow,sheetNo, ICBCBankStatementEntry.class,bankStatementMapper,cardHolderName,sourceFile,headFields); } } public void analyzeIData(String caseId) { // 储蓄卡流水 analyzeSavingsCardBS(caseId); // 信用卡流水 analyzeCreditCardBS(caseId); } private void analyzeCreditCardBS(String caseId) { List bsList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE); List entityList = HelperUtil.getEntityList(creditCardBankStatementMapper, ICBCCreditCardBankStatementEntry.class); List oaiData = importService.getOAIData(caseId, BANK_NAME); Map nameAccMap = new HashMap<>(); Map cardNumberAccMap = new HashMap<>(); Map account2CardNumberMap = new HashMap<>(); for (OpeningAccountInfo entry : oaiData) { if (StrUtil.isNotEmpty(entry.getName())) { nameAccMap.put(entry.getName(), entry); } if (StrUtil.isNotEmpty(entry.getAccountNumber())) { cardNumberAccMap.put(entry.getAccountNumber(), entry); } if (StrUtil.isNotEmpty(entry.getAccount2CardNumber())) { account2CardNumberMap.put(entry.getAccount2CardNumber(), entry); } } //去重 Set uniqueKeySet = new HashSet(); List plateNumberInfoList = new ArrayList<>(); for (ICBCCreditCardBankStatementEntry entry : entityList) { String sourceFile = entry.getSourceFile(); try { BankStatement bs = new BankStatement(); bs.setBankName(BANK_NAME); bs.setCardNumber(entry.getCardNumber()); bs.setCardHolderName(entry.getCardHolderName()); if (StrUtil.isEmpty(bs.getCardHolderName())) { // 从开户信息中获取数据 OpeningAccountInfo acc = cardNumberAccMap.getOrDefault(entry.getCardNumber(), null); if (acc != null) { bs.setCardHolderName(acc.getName()); bs.setIdCardNo(acc.getIdNo()); }else { acc = account2CardNumberMap.getOrDefault(entry.getCardNumber(), null); if (acc != null) { bs.setCardHolderName(acc.getName()); bs.setIdCardNo(acc.getIdNo()); } } }else { // 从开户信息中获取数据 OpeningAccountInfo acc = nameAccMap.getOrDefault(entry.getCardHolderName(), null); if (acc != null) { bs.setIdCardNo(acc.getIdNo()); } } bs.setCounterpartyName(entry.getCounterpartyName()); bs.setCounterpartIdCardNo(null); bs.setCounterpartyAccount(entry.getCounterpartyAccount()); bs.setSummary(entry.getSummary()); bs.setTransRemark(entry.getTransRemark()); bs.setTransCurrencyType(Constants.CURRENCY_TYPE_CHINA); bs.setTransactionInstitutions(entry.getTransactionInstitutions()); // 交易金额 String loanFlag = entry.getLoanFlag(); if (StrUtil.isNotEmpty(loanFlag)) { if (Objects.equals("借", loanFlag)) { bs.setTransactionAmount(BigDecimal.ZERO.subtract(entry.getTransactionAmount())); } else { bs.setTransactionAmount(entry.getTransactionAmount()); } } // 交易时间 2015-08-29-13.16.32.670138 String transactionTime = entry.getTransactionTime(); String pattern = "yyyy-MM-dd-HH.mm.ss"; int len = pattern.length(); transactionTime = transactionTime.substring(0, len); try { bs.setTransactionTime(DateUtil.parse(transactionTime, pattern)); } catch (Exception e) { log.error("解析时间出错", e); throw new AnalyzeDataFailedException("解析时间出错: " + e.getMessage(), e, sourceFile); } // 信用卡余额为0 bs.setBalance(entry.getUpdateBalance()); bs.setRemark("信用卡"); String md5Id = HelperUtil.generateMD5Id(bs,caseId); //未导入数据内部去重 if(HelperUtil.deduplication(md5Id,uniqueKeySet)){ continue; } bs.setId(md5Id); bs.setCaseId(caseId); try { BeanUtils.beanAttributeValueTrim(bs); } catch (Exception e) { e.printStackTrace(); } bsList.add(bs); HelperUtil.extractPlateNumber(bs,plateNumberInfoList); if (bsList.size() >= Constants.BATCH_SIZE) { List dest = HelperUtil.getDest(bsList); HelperUtil.batchInsert2Es(dest, caseId); bsList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE); } } catch (Exception e) { importResultService.record(caseId, BANK_NAME, e, sourceFile); } } uniqueKeySet.clear(); HelperUtil.batchInsertPlateNumber(plateNumberInfoList); if (!bsList.isEmpty()) { List dest = HelperUtil.getDest(bsList); HelperUtil.batchInsert2Es(dest, caseId); } } private void analyzeSavingsCardBS(String caseId) { List bsList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE); List entityList = HelperUtil.getEntityList(bankStatementMapper, ICBCBankStatementEntry.class); // Map> oaiMap = importService.getOAIMap(caseId, BANK_NAME); List oaiData = importService.getOAIData(caseId, BANK_NAME); Map nameAccMap = new HashMap<>(); Map cardNumberAccMap = new HashMap<>(); Map accountNumberAccMap = new HashMap<>(); Map account2CardNumberMap = new HashMap<>(); for (OpeningAccountInfo entry : oaiData) { if (StrUtil.isNotEmpty(entry.getName())) { nameAccMap.put(entry.getName(), entry); } if (StrUtil.isNotEmpty(entry.getAccountNumber())) { cardNumberAccMap.put(entry.getAccountNumber(), entry); } if (StrUtil.isNotEmpty(entry.getCustomerId())) { accountNumberAccMap.put(entry.getCustomerId(), entry); } if (StrUtil.isNotEmpty(entry.getAccount2CardNumber())) { account2CardNumberMap.put(entry.getAccount2CardNumber(), entry); } } Set uniqueKeySet = new HashSet(); List plateNumberInfoList = new ArrayList<>(); for (ICBCBankStatementEntry entry : entityList) { String sourceFile = entry.getSourceFile(); try { String cardNumber = entry.getCardNumber(); String accountNumber = entry.getAccountNumber(); if (StrUtil.isEmpty(entry.getCardHolderName())) { if (cardNumberAccMap.containsKey(cardNumber)) { OpeningAccountInfo info = cardNumberAccMap.get(cardNumber); entry.setCardHolderName(info.getName()); entry.setIdCardNo(info.getIdNo()); } else if (cardNumberAccMap.containsKey(accountNumber)) { OpeningAccountInfo info = cardNumberAccMap.get(accountNumber); entry.setCardHolderName(info.getName()); entry.setIdCardNo(info.getIdNo()); } else if(accountNumberAccMap.containsKey(cardNumber)){ OpeningAccountInfo info = accountNumberAccMap.get(cardNumber); entry.setCardHolderName(info.getName()); entry.setIdCardNo(info.getIdNo()); } else if(accountNumberAccMap.containsKey(accountNumber)){ OpeningAccountInfo info = accountNumberAccMap.get(accountNumber); entry.setCardHolderName(info.getName()); entry.setIdCardNo(info.getIdNo()); } else if(account2CardNumberMap.containsKey(accountNumber)){ OpeningAccountInfo info = account2CardNumberMap.get(accountNumber); entry.setCardHolderName(info.getName()); entry.setIdCardNo(info.getIdNo()); }else if(account2CardNumberMap.containsKey(cardNumber)){ OpeningAccountInfo info = account2CardNumberMap.get(cardNumber); entry.setCardHolderName(info.getName()); entry.setIdCardNo(info.getIdNo()); } }else { OpeningAccountInfo acc = nameAccMap.getOrDefault(entry.getCardHolderName(), null); if (acc != null) { entry.setIdCardNo(acc.getIdNo()); } } BankStatement bs = new BankStatement(); bs.setBankName(BANK_NAME); cardNumber = StrUtil.isEmpty(cardNumber)? accountNumber : cardNumber; bs.setCardNumber(cardNumber); bs.setCardHolderName(entry.getCardHolderName()); bs.setIdCardNo(entry.getIdCardNo() == null?"":entry.getIdCardNo()); String counterpartName = entry.getCounterpartName(); bs.setCounterpartyName(counterpartName); bs.setCounterpartyAccount(entry.getCounterpartyAccount()); bs.setCounterpartyBankName(entry.getCounterpartyBankName()); bs.setTransRemark(entry.getTransRemark()); bs.setTransCurrencyType(Constants.CURRENCY_TYPE_CHINA); bs.setTransactionInstitutions(entry.getTransactionInstitutions()); // 交易金额 String loanFlag = entry.getLoanFlag(); try { if (StrUtil.isNotEmpty(loanFlag) && StrUtil.isNotEmpty(entry.getTransactionAmount())) { BigDecimal transactionAmount = new BigDecimal(entry.getTransactionAmount()); if (transactionAmount.compareTo(BigDecimal.ZERO) < 0) { transactionAmount = BigDecimal.ZERO.subtract(transactionAmount); } if (Objects.equals("借", loanFlag)) { bs.setTransactionAmount(BigDecimal.ZERO.subtract( new BigDecimal(entry.getTransactionAmount()))); } else { bs.setTransactionAmount(transactionAmount); } } }catch (Exception e){ throw new AnalyzeDataFailedException( StrUtil.format("金额, 无法将字符串【{}】化为金额。", entry.getTransactionAmount()), e, sourceFile); } // 交易时间 2015-08-29-13.16.32.670138 String transactionTime = entry.getTransactionTime(); String transactionDate = entry.getTransactionDate(); try { if(StrUtil.isNotEmpty(transactionDate)){ String format = "yyyy-MM-ddHH.mm.ss"; if(!transactionDate.contains("-")){ format = "yyyyMMddHH.mm.ss"; } bs.setTransactionTime(DateUtil.parse(transactionDate + transactionTime, format)); }else if(StrUtil.isNotEmpty(transactionTime) && transactionTime.contains(".")){ String pattern = "yyyy-MM-dd-HH.mm.ss"; int len = pattern.length(); transactionTime = transactionTime.substring(0, len); bs.setTransactionTime(DateUtil.parse(transactionTime, pattern)); }else { } } catch (Exception e) { throw new AnalyzeDataFailedException(StrUtil.format("解析时间失败,时间:{}", transactionTime), e,entry.getSourceFile()); } bs.setBalance(new BigDecimal(entry.getBalance())); bs.setSummary(entry.getSummary()); bs.setTransChannel(entry.getTransChannel()); String md5Id = HelperUtil.generateMD5Id(bs,caseId); //未导入数据内部去重 if(HelperUtil.deduplication(md5Id,uniqueKeySet)){ continue; } bs.setSourceFile(entry.getSourceFile()); bs.setId(md5Id); bs.setCaseId(caseId); try { BeanUtils.beanAttributeValueTrim(bs); } catch (Exception e) { e.printStackTrace(); } bsList.add(bs); HelperUtil.extractPlateNumber(bs,plateNumberInfoList); if (bsList.size() >= Constants.BATCH_SIZE) { List dest = HelperUtil.getDest(bsList); HelperUtil.batchInsert2Es(dest, caseId); bsList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE); } } catch (Exception e) { importResultService.record(caseId, BANK_NAME, e, sourceFile); } } uniqueKeySet.clear(); HelperUtil.batchInsertPlateNumber(plateNumberInfoList); if (!bsList.isEmpty()) { List dest = HelperUtil.getDest(bsList); HelperUtil.batchInsert2Es(dest, caseId); } } private void readBS(String excelFileName, int sheetNo,String sourceFile) { try { // 分两种流水,一种是信用卡流水,一种是储蓄卡流水 int type = getBSType(excelFileName, sheetNo); if (type == Constants.ICBC_BS_TYPE_CREDIT_CARD) { // 信用卡流水 try (ExcelReader reader = EasyExcel.read(excelFileName).build()) { ReadSheet sheet = EasyExcel.readSheet(sheetNo) .head(ICBCCreditCardBankStatementEntry.class) .registerReadListener(HelperUtil.getReadListener( creditCardBankStatementMapper, ICBCCreditCardBankStatementEntry.class, "",sourceFile)) .build(); reader.read(sheet); } catch (Exception e) { log.error("读取excel文件失败", e); throw new ImportDataFailedException(e.getMessage(), excelFileName); } } else if (type == Constants.ICBC_BS_TYPE_DEBIT_CARD) { // 储蓄卡流水 try (ExcelReader reader = EasyExcel.read(excelFileName).build()) { ReadSheet sheet = EasyExcel.readSheet(sheetNo) .head(ICBCBankStatementEntry.class) .registerReadListener(HelperUtil.getReadListener( bankStatementMapper, ICBCBankStatementEntry.class, "",sourceFile)) .build(); reader.read(sheet); } catch (Exception e) { log.error("读取excel文件失败", e); throw new ImportDataFailedException(e.getMessage(), excelFileName); } } } catch (Exception e) { log.error("读取excel文件失败", e); throw new ImportDataFailedException(e.getMessage(), excelFileName); } } private int getBSType(String excelFilename, int sheetNo) { File file = new File(excelFilename); Cell cell = AsposeUtil.getCell(file, sheetNo, "更新后余额"); if (cell != null) { return Constants.ICBC_BS_TYPE_CREDIT_CARD; } else { Cell c = AsposeUtil.getCell(file, sheetNo, "余额"); if (c != null) { return Constants.ICBC_BS_TYPE_DEBIT_CARD; } } return -1; } }