package com.inscloudtech.bankStatementAnalysis.helper; import cn.hutool.core.date.DateUtil; import cn.hutool.core.io.FileUtil; import cn.hutool.core.util.IdUtil; import cn.hutool.core.util.NumberUtil; import cn.hutool.core.util.StrUtil; import com.alibaba.excel.EasyExcel; import com.alibaba.excel.ExcelReader; import com.alibaba.excel.context.AnalysisContext; import com.alibaba.excel.exception.ExcelAnalysisStopException; import com.alibaba.excel.read.listener.ReadListener; import com.alibaba.excel.read.metadata.ReadSheet; import com.alibaba.excel.util.ListUtils; import com.aspose.cells.Cell; import com.aspose.cells.Cells; import com.aspose.cells.Workbook; import com.aspose.cells.Worksheet; import com.baomidou.mybatisplus.core.toolkit.StringUtils; import com.inscloudtech.bankStatementAnalysis.helper.HelperUtil; import com.inscloudtech.common.constant.Constants; import com.inscloudtech.common.exception.dc.AnalyzeDataFailedException; import com.inscloudtech.common.exception.dc.ImportDataFailedException; import com.inscloudtech.common.exception.dc.StopReadException; import com.inscloudtech.common.exception.dc.TemplateNotFindException; import com.inscloudtech.common.utils.bean.BeanUtils; import com.inscloudtech.converter.ExcelStringToJavaBigDecimalConverter; import com.inscloudtech.datacenter.domain.PlateNumberInfo; import com.inscloudtech.datacenter.mapper.es.ESOpeningAccountInfoMapper; import com.inscloudtech.datacenter.mapper.es.EsCITICBankStatementMapper; import com.inscloudtech.datacenter.mapper.es.EsCITICOpeningAccountInfoMapper; import com.inscloudtech.datacenter.service.ImportResultService; import com.inscloudtech.bankStatementAnalysis.listener.CITICBankOpeningAccountInfoReadListener; import com.inscloudtech.datacenter.domain.BankStatement; import com.inscloudtech.datacenter.domain.OpeningAccountInfo; import com.inscloudtech.bankStatementAnalysis.domain.entity.CITICBankStatementEntity; import com.inscloudtech.bankStatementAnalysis.domain.entity.CITICOpeningAccountInfoEntity; import com.inscloudtech.bankStatementAnalysis.util.AsposeUtil; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import java.io.File; import java.math.BigDecimal; import java.util.*; import java.util.stream.Collectors; /** * 中信银行数据分析 */ @RequiredArgsConstructor @Component @Slf4j public class CITICDataAnalysisHelper { private final ImportResultService importResultService; private final EsCITICOpeningAccountInfoMapper esCITICOpeningAccountInfoMapper; private final EsCITICBankStatementMapper esCITICBankStatementMapper; private final ESOpeningAccountInfoMapper esOAIMapper; private final static String BANK_NAME = "中信银行"; private String caseId = ""; public void analyzeData(String caseId) throws Exception { // 开户信息分析 analyzeOAI( caseId); analyzeBS(caseId); } public void importData(File dir,String caseId) throws Exception { this.caseId = caseId; if (!dir.exists()) { return; } List fileList = FileUtil.loopFiles(dir); // // pdf // List pdfFileList = // fileList.stream().filter(f -> f.getName().endsWith(".pdf")).collect(Collectors.toList()); // // // // pdf转化为Excel // for (File f : pdfFileList) { // try { // String excelFilename = AsposeUtil.pdf2ExcelV6(f.getAbsolutePath()); // excelFile.add(new File(excelFilename)); // } catch (Exception e) { // log.error("pdf文件转excel失败:" + e.getMessage(), e); // throw new ImportDataFailedException("pdf文件转excel失败:" + f.getAbsolutePath()); // } // } List excelFile = HelperUtil.getExcelFile(fileList); for (File file : excelFile) { try { String absolutePath = file.getAbsolutePath(); // 读取文件来确认是开户信息,还是流水信息 // 这种方式是有漏洞的,一个文件,既可能包含开户信息,也可能包含流水信心 Workbook wb = new Workbook(absolutePath); for (int sheetNo = 0; sheetNo < wb.getWorksheets().getCount(); sheetNo++) { try { Cells cells = wb.getWorksheets().get(sheetNo).getCells(); String sourceFile = HelperUtil.getSourceFileName(absolutePath,BANK_NAME); Cell creditCardDateCell = AsposeUtil.getCell(cells, "入账日期"); Cell tradeCell = AsposeUtil.getCell(cells, "交易日期"); if (AsposeUtil.getCell(cells, "开户日期") != null) { if((AsposeUtil.getCell(cells, "地址") != null && !Objects.equals( Objects.requireNonNull(AsposeUtil.getCell(cells, "地址")) .getStringValue(), "IP地址"))){ // 解析开户信息 readOAI(file, cells, sheetNo); }else { try (ExcelReader excelReader = EasyExcel.read(file).build()) { ReadSheet readSheet = EasyExcel.readSheet(sheetNo) .headRowNumber(1) .head(CITICOpeningAccountInfoEntity.class) .registerReadListener(new CITICBankOpeningAccountInfoReadListener( esCITICOpeningAccountInfoMapper, null)) .build(); excelReader.read(readSheet); } catch (Exception e) { throw new ImportDataFailedException(e.getMessage(), sourceFile); } } } else if (tradeCell != null) { if(AsposeUtil.getCell(cells, "交易时间") != null){ readBSV4(wb, file, cells, sheetNo); /**20240418新信用卡流水模板*/ }else { /**20240428新流水模板*/ readCreditCard(absolutePath,sheetNo,tradeCell); } }else if(creditCardDateCell != null){ /**20240418新信用卡流水模板*/ readCreditCard(absolutePath,sheetNo,creditCardDateCell); }else { // throw new ImportDataFailedException("文件无对应模板", sourceFile); } }catch (Exception e){ importResultService.record(caseId,BANK_NAME,e); } } }catch (Exception e){ importResultService.record(caseId,BANK_NAME,e); } } } private void analyzeBS(String caseId) { List bsList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE); List entityList = HelperUtil.getEntityList(esCITICBankStatementMapper, CITICBankStatementEntity.class); List oaiList = HelperUtil.getEntityListV2(esOAIMapper, OpeningAccountInfo.class, caseId); Map> groupByName = oaiList.stream().filter(item ->StrUtil.isNotEmpty(item.getName())) .collect(Collectors.groupingBy(OpeningAccountInfo::getName)); Map> groupByAccountNumber = oaiList.stream().filter(item ->StrUtil.isNotEmpty(item.getAccountNumber())) .collect(Collectors.groupingBy(OpeningAccountInfo::getAccountNumber)); Map> groupByIdCard = oaiList.stream().filter(item ->StrUtil.isNotEmpty(item.getIdNo())) .collect(Collectors.groupingBy(OpeningAccountInfo::getIdNo)); Map nameCardNumberMap = new HashMap<>(); Map cardAndIdCardMap = new HashMap<>(); Set uniqueKeySet = new HashSet(); List plateNumberInfoList = new ArrayList<>(); for (CITICBankStatementEntity entity : entityList) { String sourceFile = entity.getSourceFile(); try { BankStatement bs = new BankStatement(); bs.setBankName(BANK_NAME); String cardNumber = entity.getCardNumber(); bs.setCardNumber(cardNumber); String customerName = entity.getCustomerName(); if(StrUtil.isEmpty(customerName)){//流水名字为空,通过卡号关联出名字 if(groupByAccountNumber.containsKey(cardNumber)){ List openingAccountInfos = groupByAccountNumber.get(cardNumber); for (OpeningAccountInfo info : openingAccountInfos) { if(StrUtil.isNotEmpty(info.getName())){ customerName = info.getName(); break; } } } } bs.setCardHolderName(customerName); if (StringUtils.isNotEmpty(cardNumber)) { nameCardNumberMap.put(customerName, cardNumber); if(groupByAccountNumber.containsKey(cardNumber)){ List openingAccountInfos = groupByAccountNumber.get(cardNumber); for (OpeningAccountInfo info : openingAccountInfos) { if(StrUtil.isNotEmpty(info.getIdNo())){ bs.setIdCardNo(info.getIdNo()); break; } } } // 开户信息中没有的卡号,根据姓名来查询 // 姓名为空,则不处理 if (StringUtils.isNotEmpty(customerName)) { if(groupByName.containsKey(customerName)){ List infoList = groupByName.get(customerName); for (OpeningAccountInfo info : infoList) { if(StrUtil.isNotEmpty(info.getIdNo())){ bs.setIdCardNo(info.getIdNo()); break; } } } } } if(StrUtil.isEmpty(bs.getIdCardNo()) && StrUtil.isNotEmpty(entity.getIdCardNo())){ bs.setIdCardNo(entity.getIdCardNo()); } if(StrUtil.isEmpty(bs.getCardHolderName()) && StrUtil.isNotEmpty(bs.getIdCardNo())){ if(groupByIdCard.containsKey(bs.getIdCardNo())){ List openingAccountInfos = groupByIdCard.get(bs.getIdCardNo()); for (OpeningAccountInfo info : openingAccountInfos) { if(StrUtil.isNotEmpty(info.getName())){ bs.setCardHolderName(info.getName()); break; } } } } // 交易时间 String transDate = entity.getTransDate(); if (StrUtil.isNotEmpty(transDate)) { // 三种情况,/ - 以及没有符号 String format = null; if (transDate.contains("/")) { format = "yyyy/MM/dd"; } else if (transDate.contains("-")) { format = "yyyy-MM-dd"; } else { format = "yyyyMMdd"; } String transTime = entity.getTransTime(); if (StrUtil.isNotEmpty(transTime)) { // 两种情况,一种是包含冒号,一种没有 if (transTime.contains(":")) { format = format + " HH:mm:ss"; } else { format = format + " HHmmss"; } // 如果不足六位,则补足 if (transTime.length() < Constants.TIME_FULL_FORMAT_LENGTH) { int cnt = Constants.TIME_FULL_FORMAT_LENGTH - transTime.length(); StringBuilder transTimeBuilder = new StringBuilder(transTime); for (int i = 0; i < cnt; i++) { transTimeBuilder.insert(0, "0"); } transTime = String.valueOf(transTimeBuilder); } try { bs.setTransactionTime(DateUtil.parse(transDate + " " + transTime, format)); } catch (Exception e) { throw new AnalyzeDataFailedException( StrUtil.format("解析交易时间异常[{}]", entity.getTransDate() + " " + entity.getTransTime()), e, entity.getSourceFile()); } } else { try { bs.setTransactionTime(DateUtil.parse(transDate, format)); } catch (Exception e) { throw new AnalyzeDataFailedException(StrUtil.format("解析交易时间异常[{}]", transDate), e,entity.getSourceFile()); } } } // 交易金额 // 需要根据借贷标记来计算 // c+ d- String creditMark = entity.getCreditMark(); if (StrUtil.isNotEmpty(creditMark)) { BigDecimal transactionAmount = NumberUtil.toBigDecimal(entity.getTransactionAmount()); if (creditMark.toLowerCase().contains("c")) { bs.setTransactionAmount(transactionAmount); } else { bs.setTransactionAmount(BigDecimal.ZERO.subtract(transactionAmount)); } } else if (entity.getCreditCard()){ // 没有借贷标志,说明交易金额是零 bs.setTransactionAmount(NumberUtil.toBigDecimal(entity.getTransactionAmount())); }else { bs.setTransactionAmount(BigDecimal.ZERO); } // 余额 bs.setBalance(NumberUtil.toBigDecimal(entity.getBalance())); // 交易对手 String counterpartName = entity.getCounterpartyName(); bs.setCounterpartyName(counterpartName); // 交易币种 bs.setTransCurrencyType(Constants.CURRENCY_TYPE_CHINA); bs.setCounterpartyAccount(entity.getCounterpartyAccount()); bs.setCounterpartyBankName(entity.getCounterpartyBankName()); bs.setSummary(entity.getSummary()); bs.setTransRemark(entity.getTransRemark()); bs.setTransChannel(entity.getTransChannel()); cardAndIdCardMap.put(bs.getCardNumber(),bs.getIdCardNo()); String md5Id = HelperUtil.generateMD5Id(bs,caseId); //未导入数据内部去重 if(HelperUtil.deduplication(md5Id,uniqueKeySet)){ continue; } bs.setSourceFile(entity.getSourceFile()); bs.setId(md5Id); bs.setCaseId(caseId); try { BeanUtils.beanAttributeValueTrim(bs); } catch (Exception e) { e.printStackTrace(); } bsList.add(bs); HelperUtil.extractPlateNumber(bs,plateNumberInfoList); if (bsList.size() >= Constants.BATCH_SIZE) { bsList = handleCardNumber(bsList, nameCardNumberMap); // 批量保存 // 保存数据库 List dest = HelperUtil.getDest(bsList); HelperUtil.batchInsert2Es(dest, caseId); bsList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE); } } catch (Exception e) { importResultService.record(caseId, BANK_NAME, e,sourceFile); } } uniqueKeySet.clear(); HelperUtil.batchInsertPlateNumber(plateNumberInfoList); if (!bsList.isEmpty()) { //注释原因该方法导致某些流水姓名字段为空 // bsList = handleCardNumber(bsList, nameCardNumberMap); List dest = HelperUtil.getDest(bsList); HelperUtil.batchInsert2Es(dest, caseId); } } /** * 为什么再次 if (StrUtil.isNotEmpty(cardNumber)) { bs.setCardNumber(no); * @param bsList * @param nameCardNumberMap * @return */ private List handleCardNumber(List bsList, Map nameCardNumberMap) { return bsList.stream() .peek(bs -> { String cardNumber = bs.getCardNumber(); if (StrUtil.isNotEmpty(cardNumber)) { String cardHolderName = bs.getCardHolderName(); if (StrUtil.isNotEmpty(cardHolderName)) { String no = nameCardNumberMap.getOrDefault(cardHolderName, null); if (StrUtil.isNotEmpty(no)) { bs.setCardNumber(no); } } } }) .collect(Collectors.toList()); } private void analyzeOAI( String caseId) { List oaiList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE); // ES中有开户信息 List entityList = HelperUtil.getEntityList(esCITICOpeningAccountInfoMapper, CITICOpeningAccountInfoEntity.class); Set uniqueKeySet = new HashSet(); for (CITICOpeningAccountInfoEntity entity : entityList) { if (StringUtils.isEmpty(entity.getCardHolderName())) { continue; } OpeningAccountInfo oai = new OpeningAccountInfo(); oai.setName(entity.getCardHolderName()); oai.setBankName(BANK_NAME); oai.setIdType(entity.getIdType()); oai.setIdNo(entity.getIdCardNo()); oai.setPhone(entity.getPhone()); oai.setAddress(entity.getAddress()); String accountNumber = StrUtil.isEmpty(entity.getAccountNumberItem())?entity.getAccountNumber():entity.getAccountNumberItem(); oai.setAccountNumber(accountNumber); oai.setStatus(entity.getStatus()); oai.setOpeningAccountDate(entity.getOpeningAccountDate()); oai.setBalance(entity.getBalance()); String md5Id = HelperUtil.generateMD5Id4OAI(oai,caseId); //未导入数据内部去重 if(HelperUtil.deduplication(md5Id,uniqueKeySet)){ continue; } oai.setId(md5Id); oai.setCaseId(caseId); oaiList.add(oai); if (oaiList.size() >= Constants.BATCH_SIZE) { List dest = HelperUtil.getDest(oaiList); HelperUtil.batchSaveOAI2Es(dest, caseId); } } if (!oaiList.isEmpty()) { List dest = HelperUtil.getDest(oaiList); HelperUtil.batchSaveOAI2Es(dest, caseId); } } public ReadListener creditCardReadListener(String sourceFile) { return new ReadListener() { List cacheList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE); @Override public void invoke(CITICBankStatementEntity entity, AnalysisContext context) { entity.setId(IdUtil.objectId()); entity.setSourceFile(sourceFile); entity.setCreditCard(true); entity.setBalance("0.0"); cacheList.add(entity); if (cacheList.size() >= Constants.BATCH_SIZE) { saveData2Es(); } } @Override public void doAfterAllAnalysed(AnalysisContext context) { if (!cacheList.isEmpty()) { saveData2Es(); } } private void saveData2Es() { esCITICBankStatementMapper.insertBatch(cacheList); cacheList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE); } }; } private void readCreditCard(String excelFileName,int sheetNum,Cell creditCardDateCell) { String sourceFile = HelperUtil.getSourceFileName(excelFileName,BANK_NAME); int headRowNumber = creditCardDateCell.getRow() + 1; try (ExcelReader reader = EasyExcel.read(excelFileName).build()) { ReadSheet sheet = EasyExcel.readSheet(sheetNum) .head(CITICBankStatementEntity.class) .headRowNumber(headRowNumber) .registerReadListener(creditCardReadListener(sourceFile)) .build(); reader.read(sheet); } catch (Exception e) { // log.error("读取私人银行流水出错:{}", e.getMessage(), e); throw new ImportDataFailedException( StrUtil.format("读取私人银行流水出错, 请检查文件【{}秒】是否正确。", sourceFile), sourceFile); } } private void readBSV4(Workbook wb, File excelFile, Cells cells, int sheetNo) throws Exception { // 表头 List headerCellList = AsposeUtil.find(cells, "交易日期"); for (Cell headerCell : headerCellList) { int headerRowNum = headerCell.getRow() + 1; boolean modified = fixWorksheetAndReadExcelFile(wb, cells, excelFile, headerCell, headerRowNum, sheetNo); if (modified) { wb = new Workbook(excelFile.getAbsolutePath()); Worksheet worksheet = wb.getWorksheets().get(sheetNo); cells = worksheet.getCells(); } } } private boolean fixWorksheetAndReadExcelFile(Workbook wb, Cells cells, File excelFile, Cell headerCell, int headRowNum, int sheetNo) throws Exception { int customerInfoRow = headerCell.getRow() - 1; boolean isModified = false; // 需要分类 // 先根据客户名来分,一种有客户名,一种没有客户名 Cell customerNameCell = AsposeUtil.getCell(cells, "客户名", Math.max(customerInfoRow, 0)); String pathname = excelFile.getAbsolutePath(); String sourceFile = HelperUtil.getSourceFileName(pathname,BANK_NAME); if (customerNameCell != null && customerNameCell.getRow() == customerInfoRow) { // 有客户名的又分为两种, // 一种是账号在外,一种是表内有客户账号 Cell cell = AsposeUtil.getCell(cells, "账号", Math.max(customerInfoRow, 0)); if (cell != null && cell.getRow() <= headRowNum - 1) { String customerName = null; // 获取客户名 String v = customerNameCell.getStringValue(); if (StrUtil.isNotEmpty(v)) { if (v.contains(":") || v.contains(":")) { v = v.replace(":", ":"); customerName = v.substring(v.indexOf(":") + 1).trim(); } else { customerName = AsposeUtil.getNextCellValue(pathname, 0, customerNameCell); } } if (cell.getRow() < headRowNum - 1) { // 一种,客户账号在表外 String cardNumber = null; // 获取账号 String value = cell.getStringValue(); if (StrUtil.isNotEmpty(value)) { if (value.contains(":") || value.contains(":")) { value = value.replace(":", ":"); cardNumber = value.substring(value.indexOf(":") + 1).trim(); } else { cardNumber = AsposeUtil.getNextCellValue(pathname, 0, cell); } } Cell c = AsposeUtil.getCell(cells, "客户名", headerCell.getRow() + 1); int endRowNum = -1; if (c != null) { endRowNum = c.getRow(); } else { endRowNum = cells.getMaxRow(); } if (StrUtil.isNotEmpty(cardNumber)) { fix(wb, pathname, cells, headerCell, endRowNum, "客户账号", cardNumber); isModified = true; excelFile = new File(pathname); } if (StrUtil.isNotEmpty(customerName)) { fix(wb, pathname, cells, headerCell, endRowNum, "客户名称", customerName); isModified = true; excelFile = new File(pathname); } readBSV5(pathname, headRowNum, endRowNum, sheetNo); } else { // 一种,客户账号在表内 try { // 分析 Cell c = AsposeUtil.getCell(cells, "客户名", headerCell.getRow() + 1); int endRowNum = -1; if (c != null) { endRowNum = c.getRow(); } else { endRowNum = cells.getMaxRow(); } if (StrUtil.isNotEmpty(customerName)) { // 添加一列 fix(wb, pathname, cells, headerCell, endRowNum, "客户名称", customerName); isModified = true; excelFile = new File(pathname);//?? } readBSV5(pathname, headRowNum, endRowNum, sheetNo); } catch (Exception e) { log.error("解析Excel文件失败.", e); throw new ImportDataFailedException(e.getMessage(), pathname); } } } } else { if(customerNameCell == null){ /**20240418新模板*/ customerNameCell = AsposeUtil.getCell(cells, "客户名", Math.max(customerInfoRow - 1, 0)); if(customerNameCell != null){ fix(wb, pathname, cells, headerCell, cells.getMaxRow(), "客户名称", cells.get(customerInfoRow, 1).getStringValue()); readBSV5(excelFile.getAbsolutePath(), headRowNum, cells.getMaxRow() + 1, sheetNo); }else { throw new TemplateNotFindException(sourceFile); } }else { readBSV5(excelFile.getAbsolutePath(), headRowNum, cells.getMaxRow() + 1, sheetNo); } } return isModified; } private void readBSV5(String pathname, int headRowNumber, int endRowNum, int sheetNo) { String sourceFile = HelperUtil.getSourceFileName(pathname,BANK_NAME); try (ExcelReader reader = EasyExcel.read(pathname).build()) { ReadSheet rs = EasyExcel.readSheet(sheetNo) .headRowNumber(headRowNumber) .head(CITICBankStatementEntity.class) .registerReadListener(HelperUtil.getReadListener( esCITICBankStatementMapper, CITICBankStatementEntity.class, headRowNumber, endRowNum ,pathname)) .build(); reader.read(rs); } catch (Exception e) { if (e instanceof ExcelAnalysisStopException) { return; } log.error("解析Excel文件失败.", e); throw new ImportDataFailedException(e.getMessage(), sourceFile); } } private void fix(Workbook wb, String filename, Cells cells, Cell headerCell, int endRowNum, String cellName, String cellValue) throws Exception { cells.insertColumn(0, true); cells.get(headerCell.getRow(), 0).setValue(cellName); for (int i = headerCell.getRow() + 1; i <= endRowNum; i++) { cells.get(i, 0).setValue(cellValue); } wb.save(filename); } /** * 解析开户信息 */ private void readOAI(File file, Cells cells, int sheetNo) { // 获取表头信息 try { // 身份证号 String idCarNo = null; Cell idCell = AsposeUtil.getCell(cells, "证件号码"); if (idCell != null) { int headRowNumber = idCell.getRow() + 1; int col = idCell.getColumn(); idCarNo = cells.get(headRowNumber, col).getStringValue(); } Cell cell = AsposeUtil.getCell(cells, "客户号"); if (cell != null) { int headRowNumber = cell.getRow() + 1; try (ExcelReader excelReader = EasyExcel.read(file).build()) { ReadSheet readSheet = EasyExcel.readSheet(sheetNo) .headRowNumber(headRowNumber) .head(CITICOpeningAccountInfoEntity.class) .registerConverter(new ExcelStringToJavaBigDecimalConverter()) .registerReadListener(new CITICBankOpeningAccountInfoReadListener( esCITICOpeningAccountInfoMapper, idCarNo)) .build(); excelReader.read(readSheet); } catch (Exception e) { log.error("解析Excel文件失败.", e); throw new ImportDataFailedException(e.getMessage(), file.getAbsolutePath()); } } } catch (Exception e) { log.error("解析Excel文件失败.", e); throw new ImportDataFailedException(e.getMessage(), file.getAbsolutePath()); } } }