You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

564 lines
25 KiB

package com.inscloudtech.bankStatementAnalysis.helper;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSON;
import cn.hutool.json.JSONObject;
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelReader;
import com.alibaba.excel.read.metadata.ReadSheet;
import com.alibaba.excel.util.ListUtils;
import com.alibaba.excel.util.StringUtils;
import com.aspose.cells.Cell;
import com.aspose.cells.Cells;
import com.aspose.cells.Workbook;
import com.aspose.cells.Worksheet;
import com.inscloudtech.bankStatementAnalysis.domain.HeadField;
import com.inscloudtech.bankStatementAnalysis.helper.HelperUtil;
import com.inscloudtech.bankStatementAnalysis.service.ImportService;
import com.inscloudtech.common.constant.BankStatementConstants;
import com.inscloudtech.common.constant.Constants;
import com.inscloudtech.common.exception.dc.AnalyzeDataFailedException;
import com.inscloudtech.common.exception.dc.ImportDataFailedException;
import com.inscloudtech.common.exception.dc.TemplateNotFindException;
import com.inscloudtech.common.utils.bean.BeanUtils;
import com.inscloudtech.common.utils.file.FileUtils;
import com.inscloudtech.datacenter.domain.PlateNumberInfo;
import com.inscloudtech.bankStatementAnalysis.mapper.EsICBCBankStatementMapper;
import com.inscloudtech.bankStatementAnalysis.mapper.EsICBCCreditCardBankStatementMapper;
import com.inscloudtech.datacenter.service.ImportResultService;
import com.inscloudtech.datacenter.domain.BankStatement;
import com.inscloudtech.datacenter.domain.OpeningAccountInfo;
import com.inscloudtech.bankStatementAnalysis.domain.entity.ICBCBankStatementEntry;
import com.inscloudtech.bankStatementAnalysis.domain.entity.ICBCCreditCardBankStatementEntry;
import com.inscloudtech.bankStatementAnalysis.util.AsposeUtil;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import java.io.File;
import java.math.BigDecimal;
import java.util.*;
/**
* 工商银行数据分析
*/
@Slf4j
@RequiredArgsConstructor
@Component
public class ICBCDataAnalysisHelper {
private final ImportService importService;
private final ImportResultService importResultService;
private final EsICBCBankStatementMapper bankStatementMapper;
private final EsICBCCreditCardBankStatementMapper creditCardBankStatementMapper;
private final static String BANK_NAME = "工商银行";
public void importData(File dir,String caseId) throws Exception {
List<File> fileList = FileUtil.loopFiles(dir);
if (fileList == null) {
return;
}
List<File> excelFileList = HelperUtil.getExcelFile(fileList);
// List<File> pdfFileList = fileList.stream()
// .filter(file -> FileUtil.extName(file).equalsIgnoreCase("pdf"))
// .collect(Collectors.toList());
// Map<String, List<File>> groupByPath = excelFileList.stream().collect(Collectors.groupingBy(File::getAbsolutePath));
// for (File file : pdfFileList) {
// String excelFilename = AsposeUtil.pdf2ExcelV6(file.getAbsolutePath());
// if(groupByPath.containsKey(excelFilename)){
// continue;
// }
// excelFileList.add(new File(excelFilename));
// }
for (File f : excelFileList) {
try {
String absolutePath = f.getAbsolutePath();
Workbook wb = new Workbook(absolutePath);
int count = wb.getWorksheets().getCount();
String sourceFile = HelperUtil.getSourceFileName(absolutePath, BANK_NAME);
for (int sheetNo = 0; sheetNo < count; sheetNo++) {
try {
Worksheet worksheet = wb.getWorksheets().get(sheetNo);
Cells cells = worksheet.getCells();
Cell contentCell = cells.get(0, 0);
if (contentCell.getValue() == null) {//空白excel
continue;
}
String worksheetName = worksheet.getName();
Cell cell420240418 = AsposeUtil.getCell(cells, "入帐日期");
Cell khCell = AsposeUtil.getCell(cells, "开户日期");
Cell tradeCell = AsposeUtil.getCell(cells, "交易时间戳");
Cell khhzh = AsposeUtil.getCell(cells, "卡号或帐号");
String withSheetName = sourceFile + BankStatementConstants.NAME_WITH_SHEET_NAME + worksheetName;
/**20240418新增模板*/
if (cell420240418 != null) {
readBSFor20240418(cells,absolutePath, sheetNo,withSheetName);
}else if (khCell != null) {
// 判断是否是开户信息
int rowNumber = khCell.getRow() + 1;
importService.readOAIData(absolutePath, rowNumber,
sheetNo, caseId, BANK_NAME, withSheetName);
} else if (tradeCell != null) { // 流水信息
readBS(absolutePath, sheetNo,withSheetName);
} else if (khhzh != null) { // 开户
importService.readOAIData(absolutePath, khhzh.getRow() + 1,
sheetNo, caseId, BANK_NAME, withSheetName);
} else {
throw new TemplateNotFindException(withSheetName);
}
} catch (Exception e) {
importResultService.record(caseId, BANK_NAME, e);
}
}
} catch (Exception e) {
importResultService.record(caseId, BANK_NAME, e);
}
}
}
private void readBSFor20240418(Cells cells ,String absolutePath, int sheetNo,String sourceFile) {
// int startRow = dateCell.getRow();
// Cell cardHolderNameCell = cells.get(startRow - 1, 1);
// if(null == cardHolderNameCell || StrUtil.isEmpty(cardHolderNameCell.getStringValue())){
// throw new ImportDataFailedException("读取持卡人信息异常", sourceFile);
// }
List<JSONObject> maps = new LinkedList<>();
int maxRow = cells.getMaxRow();
List<HeadField> headFields = new ArrayList<>();
headFields.add(new HeadField("transactionAmount","发生额"));
headFields.add(new HeadField("balance","余额"));
headFields.add(new HeadField("accountNumber","商密二级,帐号"));
for(int i = 0; i < maxRow; i++){
Cell dateCell = cells.get(i, 2);
String stringValue = dateCell.getStringValue();
if (StrUtil.isEmpty(stringValue)) {
continue;
}
if(stringValue.equals("入帐日期")){
int startRow = dateCell.getRow();
Cell cardHolderNameCell = cells.get(startRow - 1, 1);
if(null == cardHolderNameCell || StrUtil.isEmpty(cardHolderNameCell.getStringValue())){
throw new ImportDataFailedException("读取持卡人信息异常", sourceFile);
}
String cardHolderName = cardHolderNameCell.getStringValue().trim();
JSONObject jsonObject = new JSONObject();
jsonObject.putOnce("startRow",i);
jsonObject.putOnce("cardHolderName",cardHolderName);
maps.add(jsonObject);
}
}
int size = maps.size();
for(int i = 0; i < size; i++){
JSONObject jsonObject = maps.get(i);
String cardHolderName = jsonObject.getStr("cardHolderName");
int headRowNumber = jsonObject.getInt("startRow") + 1;
int endRow = maxRow -1;
if(i + 1 < size){
JSONObject nextRow = maps.get(i + 1);
endRow = nextRow.getInt("startRow") - 1;
}
if(endRow - headRowNumber <= 1){
continue;
}
importService.readMultiplePersonAndMultipleHeadBankStatement(absolutePath,headRowNumber,endRow,sheetNo,
ICBCBankStatementEntry.class,bankStatementMapper,cardHolderName,sourceFile,headFields);
}
}
public void analyzeIData(String caseId) {
// 储蓄卡流水
analyzeSavingsCardBS(caseId);
// 信用卡流水
analyzeCreditCardBS(caseId);
}
private void analyzeCreditCardBS(String caseId) {
List<BankStatement> bsList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
List<ICBCCreditCardBankStatementEntry> entityList =
HelperUtil.getEntityList(creditCardBankStatementMapper, ICBCCreditCardBankStatementEntry.class);
List<OpeningAccountInfo> oaiData = importService.getOAIData(caseId, BANK_NAME);
Map<String, OpeningAccountInfo> nameAccMap = new HashMap<>();
Map<String, OpeningAccountInfo> cardNumberAccMap = new HashMap<>();
Map<String, OpeningAccountInfo> account2CardNumberMap = new HashMap<>();
for (OpeningAccountInfo entry : oaiData) {
if (StrUtil.isNotEmpty(entry.getName())) {
nameAccMap.put(entry.getName(), entry);
}
if (StrUtil.isNotEmpty(entry.getAccountNumber())) {
cardNumberAccMap.put(entry.getAccountNumber(), entry);
}
if (StrUtil.isNotEmpty(entry.getAccount2CardNumber())) {
account2CardNumberMap.put(entry.getAccount2CardNumber(), entry);
}
}
//去重
Set<String> uniqueKeySet = new HashSet();
List<PlateNumberInfo> plateNumberInfoList = new ArrayList<>();
for (ICBCCreditCardBankStatementEntry entry : entityList) {
String sourceFile = entry.getSourceFile();
try {
BankStatement bs = new BankStatement();
bs.setBankName(BANK_NAME);
bs.setCardNumber(entry.getCardNumber());
bs.setCardHolderName(entry.getCardHolderName());
if (StrUtil.isEmpty(bs.getCardHolderName())) {
// 从开户信息中获取数据
OpeningAccountInfo acc = cardNumberAccMap.getOrDefault(entry.getCardNumber(), null);
if (acc != null) {
bs.setCardHolderName(acc.getName());
bs.setIdCardNo(acc.getIdNo());
}else {
acc = account2CardNumberMap.getOrDefault(entry.getCardNumber(), null);
if (acc != null) {
bs.setCardHolderName(acc.getName());
bs.setIdCardNo(acc.getIdNo());
}
}
}else {
// 从开户信息中获取数据
OpeningAccountInfo acc = nameAccMap.getOrDefault(entry.getCardHolderName(), null);
if (acc != null) {
bs.setIdCardNo(acc.getIdNo());
}
}
bs.setCounterpartyName(entry.getCounterpartyName());
bs.setCounterpartIdCardNo(null);
bs.setCounterpartyAccount(entry.getCounterpartyAccount());
bs.setSummary(entry.getSummary());
bs.setTransRemark(entry.getTransRemark());
bs.setTransCurrencyType(Constants.CURRENCY_TYPE_CHINA);
bs.setTransactionInstitutions(entry.getTransactionInstitutions());
// 交易金额
String loanFlag = entry.getLoanFlag();
if (StrUtil.isNotEmpty(loanFlag)) {
if (Objects.equals("借", loanFlag)) {
bs.setTransactionAmount(BigDecimal.ZERO.subtract(entry.getTransactionAmount()));
} else {
bs.setTransactionAmount(entry.getTransactionAmount());
}
}
// 交易时间 2015-08-29-13.16.32.670138
String transactionTime = entry.getTransactionTime();
String pattern = "yyyy-MM-dd-HH.mm.ss";
int len = pattern.length();
transactionTime = transactionTime.substring(0, len);
try {
bs.setTransactionTime(DateUtil.parse(transactionTime, pattern));
} catch (Exception e) {
log.error("解析时间出错", e);
throw new AnalyzeDataFailedException("解析时间出错: " + e.getMessage(), e, sourceFile);
}
// 信用卡余额为0
bs.setBalance(entry.getUpdateBalance());
bs.setRemark("信用卡");
String md5Id = HelperUtil.generateMD5Id(bs,caseId);
//未导入数据内部去重
if(HelperUtil.deduplication(md5Id,uniqueKeySet)){
continue;
}
bs.setId(md5Id);
bs.setCaseId(caseId);
try {
BeanUtils.beanAttributeValueTrim(bs);
} catch (Exception e) {
e.printStackTrace();
}
bsList.add(bs);
HelperUtil.extractPlateNumber(bs,plateNumberInfoList);
if (bsList.size() >= Constants.BATCH_SIZE) {
List<BankStatement> dest = HelperUtil.getDest(bsList);
HelperUtil.batchInsert2Es(dest, caseId);
bsList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
}
} catch (Exception e) {
importResultService.record(caseId, BANK_NAME, e, sourceFile);
}
}
uniqueKeySet.clear();
HelperUtil.batchInsertPlateNumber(plateNumberInfoList);
if (!bsList.isEmpty()) {
List<BankStatement> dest = HelperUtil.getDest(bsList);
HelperUtil.batchInsert2Es(dest, caseId);
}
}
private void analyzeSavingsCardBS(String caseId) {
List<BankStatement> bsList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
List<ICBCBankStatementEntry> entityList =
HelperUtil.getEntityList(bankStatementMapper, ICBCBankStatementEntry.class);
// Map<String, List<OpeningAccountInfo>> oaiMap = importService.getOAIMap(caseId, BANK_NAME);
List<OpeningAccountInfo> oaiData = importService.getOAIData(caseId, BANK_NAME);
Map<String, OpeningAccountInfo> nameAccMap = new HashMap<>();
Map<String, OpeningAccountInfo> cardNumberAccMap = new HashMap<>();
Map<String, OpeningAccountInfo> accountNumberAccMap = new HashMap<>();
Map<String, OpeningAccountInfo> account2CardNumberMap = new HashMap<>();
for (OpeningAccountInfo entry : oaiData) {
if (StrUtil.isNotEmpty(entry.getName())) {
nameAccMap.put(entry.getName(), entry);
}
if (StrUtil.isNotEmpty(entry.getAccountNumber())) {
cardNumberAccMap.put(entry.getAccountNumber(), entry);
}
if (StrUtil.isNotEmpty(entry.getCustomerId())) {
accountNumberAccMap.put(entry.getCustomerId(), entry);
}
if (StrUtil.isNotEmpty(entry.getAccount2CardNumber())) {
account2CardNumberMap.put(entry.getAccount2CardNumber(), entry);
}
}
Set<String> uniqueKeySet = new HashSet();
List<PlateNumberInfo> plateNumberInfoList = new ArrayList<>();
for (ICBCBankStatementEntry entry : entityList) {
String sourceFile = entry.getSourceFile();
try {
String cardNumber = entry.getCardNumber();
String accountNumber = entry.getAccountNumber();
if (StrUtil.isEmpty(entry.getCardHolderName())) {
if (cardNumberAccMap.containsKey(cardNumber)) {
OpeningAccountInfo info = cardNumberAccMap.get(cardNumber);
entry.setCardHolderName(info.getName());
entry.setIdCardNo(info.getIdNo());
} else if (cardNumberAccMap.containsKey(accountNumber)) {
OpeningAccountInfo info = cardNumberAccMap.get(accountNumber);
entry.setCardHolderName(info.getName());
entry.setIdCardNo(info.getIdNo());
} else if(accountNumberAccMap.containsKey(cardNumber)){
OpeningAccountInfo info = accountNumberAccMap.get(cardNumber);
entry.setCardHolderName(info.getName());
entry.setIdCardNo(info.getIdNo());
} else if(accountNumberAccMap.containsKey(accountNumber)){
OpeningAccountInfo info = accountNumberAccMap.get(accountNumber);
entry.setCardHolderName(info.getName());
entry.setIdCardNo(info.getIdNo());
} else if(account2CardNumberMap.containsKey(accountNumber)){
OpeningAccountInfo info = account2CardNumberMap.get(accountNumber);
entry.setCardHolderName(info.getName());
entry.setIdCardNo(info.getIdNo());
}else if(account2CardNumberMap.containsKey(cardNumber)){
OpeningAccountInfo info = account2CardNumberMap.get(cardNumber);
entry.setCardHolderName(info.getName());
entry.setIdCardNo(info.getIdNo());
}
}else {
OpeningAccountInfo acc = nameAccMap.getOrDefault(entry.getCardHolderName(), null);
if (acc != null) {
entry.setIdCardNo(acc.getIdNo());
}
}
BankStatement bs = new BankStatement();
bs.setBankName(BANK_NAME);
cardNumber = StrUtil.isEmpty(cardNumber)? accountNumber : cardNumber;
bs.setCardNumber(cardNumber);
bs.setCardHolderName(entry.getCardHolderName());
bs.setIdCardNo(entry.getIdCardNo() == null?"":entry.getIdCardNo());
String counterpartName = entry.getCounterpartName();
bs.setCounterpartyName(counterpartName);
bs.setCounterpartyAccount(entry.getCounterpartyAccount());
bs.setCounterpartyBankName(entry.getCounterpartyBankName());
bs.setTransRemark(entry.getTransRemark());
bs.setTransCurrencyType(Constants.CURRENCY_TYPE_CHINA);
bs.setTransactionInstitutions(entry.getTransactionInstitutions());
// 交易金额
String loanFlag = entry.getLoanFlag();
try {
if (StrUtil.isNotEmpty(loanFlag) && StrUtil.isNotEmpty(entry.getTransactionAmount())) {
BigDecimal transactionAmount = new BigDecimal(entry.getTransactionAmount());
if (transactionAmount.compareTo(BigDecimal.ZERO) < 0) {
transactionAmount = BigDecimal.ZERO.subtract(transactionAmount);
}
if (Objects.equals("借", loanFlag)) {
bs.setTransactionAmount(BigDecimal.ZERO.subtract( new BigDecimal(entry.getTransactionAmount())));
} else {
bs.setTransactionAmount(transactionAmount);
}
}
}catch (Exception e){
throw new AnalyzeDataFailedException(
StrUtil.format("金额, 无法将字符串【{}】化为金额。", entry.getTransactionAmount()), e, sourceFile);
}
// 交易时间 2015-08-29-13.16.32.670138
String transactionTime = entry.getTransactionTime();
String transactionDate = entry.getTransactionDate();
try {
if(StrUtil.isNotEmpty(transactionDate)){
String format = "yyyy-MM-ddHH.mm.ss";
if(!transactionDate.contains("-")){
format = "yyyyMMddHH.mm.ss";
}
bs.setTransactionTime(DateUtil.parse(transactionDate + transactionTime, format));
}else if(StrUtil.isNotEmpty(transactionTime) && transactionTime.contains(".")){
String pattern = "yyyy-MM-dd-HH.mm.ss";
int len = pattern.length();
transactionTime = transactionTime.substring(0, len);
bs.setTransactionTime(DateUtil.parse(transactionTime, pattern));
}else {
}
} catch (Exception e) {
throw new AnalyzeDataFailedException(StrUtil.format("解析时间失败,时间:{}", transactionTime), e,entry.getSourceFile());
}
bs.setBalance(new BigDecimal(entry.getBalance()));
bs.setSummary(entry.getSummary());
bs.setTransChannel(entry.getTransChannel());
String md5Id = HelperUtil.generateMD5Id(bs,caseId);
//未导入数据内部去重
if(HelperUtil.deduplication(md5Id,uniqueKeySet)){
continue;
}
bs.setSourceFile(entry.getSourceFile());
bs.setId(md5Id);
bs.setCaseId(caseId);
try {
BeanUtils.beanAttributeValueTrim(bs);
} catch (Exception e) {
e.printStackTrace();
}
bsList.add(bs);
HelperUtil.extractPlateNumber(bs,plateNumberInfoList);
if (bsList.size() >= Constants.BATCH_SIZE) {
List<BankStatement> dest = HelperUtil.getDest(bsList);
HelperUtil.batchInsert2Es(dest, caseId);
bsList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
}
} catch (Exception e) {
importResultService.record(caseId, BANK_NAME, e, sourceFile);
}
}
uniqueKeySet.clear();
HelperUtil.batchInsertPlateNumber(plateNumberInfoList);
if (!bsList.isEmpty()) {
List<BankStatement> dest = HelperUtil.getDest(bsList);
HelperUtil.batchInsert2Es(dest, caseId);
}
}
private void readBS(String excelFileName, int sheetNo,String sourceFile) {
try {
// 分两种流水,一种是信用卡流水,一种是储蓄卡流水
int type = getBSType(excelFileName, sheetNo);
if (type == Constants.ICBC_BS_TYPE_CREDIT_CARD) { // 信用卡流水
try (ExcelReader reader = EasyExcel.read(excelFileName).build()) {
ReadSheet sheet = EasyExcel.readSheet(sheetNo)
.head(ICBCCreditCardBankStatementEntry.class)
.registerReadListener(HelperUtil.getReadListener(
creditCardBankStatementMapper,
ICBCCreditCardBankStatementEntry.class,
"",sourceFile))
.build();
reader.read(sheet);
} catch (Exception e) {
log.error("读取excel文件失败", e);
throw new ImportDataFailedException(e.getMessage(), excelFileName);
}
} else if (type == Constants.ICBC_BS_TYPE_DEBIT_CARD) { // 储蓄卡流水
try (ExcelReader reader = EasyExcel.read(excelFileName).build()) {
ReadSheet sheet = EasyExcel.readSheet(sheetNo)
.head(ICBCBankStatementEntry.class)
.registerReadListener(HelperUtil.getReadListener(
bankStatementMapper, ICBCBankStatementEntry.class, "",sourceFile))
.build();
reader.read(sheet);
} catch (Exception e) {
log.error("读取excel文件失败", e);
throw new ImportDataFailedException(e.getMessage(), excelFileName);
}
}
} catch (Exception e) {
log.error("读取excel文件失败", e);
throw new ImportDataFailedException(e.getMessage(), excelFileName);
}
}
private int getBSType(String excelFilename, int sheetNo) {
File file = new File(excelFilename);
Cell cell = AsposeUtil.getCell(file, sheetNo, "更新后余额");
if (cell != null) {
return Constants.ICBC_BS_TYPE_CREDIT_CARD;
} else {
Cell c = AsposeUtil.getCell(file, sheetNo, "余额");
if (c != null) {
return Constants.ICBC_BS_TYPE_DEBIT_CARD;
}
}
return -1;
}
}