You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1555 lines
62 KiB
1555 lines
62 KiB
package com.inscloudtech.bankStatementAnalysis.helper;
|
|
|
|
import cn.hutool.core.date.DateUtil;
|
|
import cn.hutool.core.io.FileUtil;
|
|
import cn.hutool.core.util.IdUtil;
|
|
import cn.hutool.core.util.NumberUtil;
|
|
import cn.hutool.core.util.StrUtil;
|
|
import com.alibaba.excel.EasyExcel;
|
|
import com.alibaba.excel.ExcelReader;
|
|
import com.alibaba.excel.context.AnalysisContext;
|
|
import com.alibaba.excel.exception.ExcelAnalysisStopException;
|
|
import com.alibaba.excel.read.listener.ReadListener;
|
|
import com.alibaba.excel.read.metadata.ReadSheet;
|
|
import com.alibaba.excel.util.ListUtils;
|
|
import com.aspose.cells.*;
|
|
import com.baomidou.mybatisplus.core.toolkit.StringUtils;
|
|
|
|
|
|
import com.inscloudtech.bankStatementAnalysis.helper.HelperUtil;
|
|
import com.inscloudtech.bankStatementAnalysis.mapper.*;
|
|
import com.inscloudtech.common.constant.BankStatementConstants;
|
|
import com.inscloudtech.common.constant.Constants;
|
|
import com.inscloudtech.common.exception.dc.AnalyzeDataFailedException;
|
|
import com.inscloudtech.common.exception.dc.ImportDataFailedException;
|
|
import com.inscloudtech.common.exception.dc.TemplateNotFindException;
|
|
import com.inscloudtech.common.utils.bean.BeanUtils;
|
|
import com.inscloudtech.common.utils.file.FileUtils;
|
|
import com.inscloudtech.datacenter.domain.PlateNumberInfo;
|
|
import com.inscloudtech.datacenter.mapper.es.*;
|
|
import com.inscloudtech.datacenter.service.ImportResultService;
|
|
import com.inscloudtech.bankStatementAnalysis.listener.ReadCCBCurrentAccountInfoListener;
|
|
import com.inscloudtech.datacenter.domain.BankStatement;
|
|
import com.inscloudtech.datacenter.domain.OpeningAccountInfo;
|
|
|
|
import com.inscloudtech.bankStatementAnalysis.domain.entity.*;
|
|
import com.inscloudtech.bankStatementAnalysis.util.AnalyzeFileHelper;
|
|
import com.inscloudtech.bankStatementAnalysis.util.AsposeUtil;
|
|
|
|
import lombok.Data;
|
|
import lombok.RequiredArgsConstructor;
|
|
import lombok.extern.slf4j.Slf4j;
|
|
import org.apache.poi.openxml4j.util.ZipSecureFile;
|
|
import org.springframework.stereotype.Component;
|
|
|
|
import java.io.File;
|
|
import java.math.BigDecimal;
|
|
import java.util.*;
|
|
import java.util.concurrent.atomic.AtomicInteger;
|
|
import java.util.stream.Collectors;
|
|
import com.alibaba.excel.converters.Converter;
|
|
/**
|
|
* 建设银行数据分析
|
|
*/
|
|
@Slf4j
|
|
@RequiredArgsConstructor
|
|
@Component
|
|
public class CCBDataAnalysisHelper {
|
|
private final ImportResultService importResultService;
|
|
private final EsCCBCurrentAccountInfoMapper accountInfoMapper;
|
|
private final EsCCBRegularAccountInfoMapper esCCBRegularAccountInfoMapper;
|
|
private final EsCCBElectronicCashAccountInfoMapper esCCBElectronicCashAccountInfoMapper;
|
|
private final EsCCBElectronicCashBankStatementMapper esCCBElectronicCashBankStatementMapper;
|
|
private final EsCCBRegularBankStatementMapper esCCBRegularBankStatementMapper;
|
|
private final EsCCBCurrentBankStatementMapper esCCBCurrentBankStatementMapper;
|
|
private final static String BANK_NAME = "建设银行";
|
|
|
|
public void analyzeData(String caseId) {
|
|
analyzeOAI(caseId);
|
|
analyzeBS(caseId);
|
|
}
|
|
|
|
/**
|
|
* 开户信息分析
|
|
*/
|
|
private void analyzeOAI(String caseId) {
|
|
analyzeElectronicCashOpeningAccountInfo(caseId);
|
|
analyzeOtherOpeningAccountInfo(caseId);
|
|
}
|
|
|
|
private void analyzeElectronicCashOpeningAccountInfo(String caseId) {
|
|
|
|
List<OpeningAccountInfo> oaiList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
|
|
List<CCBElectronicCashAccountInfoEntity> entityList = HelperUtil.getEntityList(
|
|
esCCBElectronicCashAccountInfoMapper, CCBElectronicCashAccountInfoEntity.class);
|
|
Set<String> uniqueKeySet = new HashSet();
|
|
for (CCBElectronicCashAccountInfoEntity entity : entityList) {
|
|
|
|
String cardHolderName = entity.getCardHolderName();
|
|
if (StringUtils.isEmpty(cardHolderName)) {
|
|
continue;
|
|
}
|
|
|
|
OpeningAccountInfo oai = new OpeningAccountInfo();
|
|
oai.setName(cardHolderName);
|
|
oai.setBankName(BANK_NAME);
|
|
oai.setAccountNumber(entity.getCardNumber());
|
|
oai.setIdType(entity.getIdType());
|
|
oai.setIdNo(entity.getIdCardNo());
|
|
|
|
String md5Id = HelperUtil.generateMD5Id4OAI(oai,caseId);
|
|
//未导入数据内部去重
|
|
if(HelperUtil.deduplication(md5Id,uniqueKeySet)){
|
|
continue;
|
|
}
|
|
|
|
oai.setId(md5Id);
|
|
oai.setCaseId(caseId);
|
|
oaiList.add(oai);
|
|
if (oaiList.size() >= Constants.BATCH_SIZE) {
|
|
List<OpeningAccountInfo> dest = HelperUtil.getDest(oaiList);
|
|
HelperUtil.batchSaveOAI2Es(dest, caseId);
|
|
|
|
oaiList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
}
|
|
}
|
|
|
|
if (!oaiList.isEmpty()) {
|
|
HelperUtil.batchSaveOAI2Es(oaiList, caseId);
|
|
}
|
|
}
|
|
|
|
private void analyzeOtherOpeningAccountInfo(String caseId) {
|
|
|
|
List<OpeningAccountInfo> oaiList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
|
|
List<CCBCurrentAccountInfoEntity> entityList =
|
|
HelperUtil.getEntityList(accountInfoMapper, CCBCurrentAccountInfoEntity.class);
|
|
Set<String> uniqueKeySet = new HashSet();
|
|
for (CCBCurrentAccountInfoEntity entity : entityList) {
|
|
|
|
String cardHolderName = entity.getCardHolderName();
|
|
if (StringUtils.isEmpty(cardHolderName)) {
|
|
continue;
|
|
}
|
|
|
|
OpeningAccountInfo oai = getOpeningAccountInfo(BANK_NAME, entity);
|
|
|
|
String md5Id = HelperUtil.generateMD5Id4OAI(oai,caseId);
|
|
//未导入数据内部去重
|
|
if(HelperUtil.deduplication(md5Id,uniqueKeySet)){
|
|
continue;
|
|
}
|
|
|
|
oai.setId(md5Id);
|
|
oai.setCaseId(caseId);
|
|
oaiList.add(oai);
|
|
if (oaiList.size() >= Constants.BATCH_SIZE) {
|
|
List<OpeningAccountInfo> dest = HelperUtil.getDest(oaiList);
|
|
HelperUtil.batchSaveOAI2Es(dest, caseId);
|
|
|
|
oaiList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
}
|
|
}
|
|
|
|
if (!oaiList.isEmpty()) {
|
|
HelperUtil.batchSaveOAI2Es(oaiList, caseId);
|
|
}
|
|
}
|
|
|
|
private static OpeningAccountInfo getOpeningAccountInfo(String bankName, CCBCurrentAccountInfoEntity entity) {
|
|
OpeningAccountInfo oai = new OpeningAccountInfo();
|
|
oai.setName(entity.getCardHolderName());
|
|
oai.setBankName(BANK_NAME);
|
|
oai.setAccountNumber(entity.getCardNumber());
|
|
oai.setIdType(entity.getIdType());
|
|
oai.setIdNo(entity.getIdCardNo());
|
|
oai.setOpeningAccountDate(entity.getOpeningAccountDate());
|
|
oai.setClosingDate(entity.getClosingDate());
|
|
oai.setStatus(entity.getStatus());
|
|
oai.setAccountOpeningInstitution(entity.getAccountOpeningInstitution());
|
|
oai.setBalance(
|
|
StrUtil.isEmpty(entity.getBalance()) ? BigDecimal.ZERO : NumberUtil.toBigDecimal(entity.getBalance()));
|
|
oai.setFreezeInfo(entity.getFreezeInfo());
|
|
|
|
return oai;
|
|
}
|
|
|
|
/**
|
|
* 分析交易流水
|
|
*/
|
|
private void analyzeBS(String caseId) {
|
|
analyzeCurrentBS(caseId);
|
|
analyzeRegularBS(caseId);
|
|
analyzeElectronicCashBS(caseId);
|
|
}
|
|
|
|
private void analyzeElectronicCashBS(String caseId) {
|
|
|
|
List<BankStatement> bsList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
|
|
List<CCBElectronicCashBankStatementEntity> entityList = HelperUtil.getEntityList(
|
|
esCCBElectronicCashBankStatementMapper, CCBElectronicCashBankStatementEntity.class);
|
|
// 根据名称去查找基本信息
|
|
List<CCBElectronicCashAccountInfoEntity> infoList = HelperUtil.getEntityList(esCCBElectronicCashAccountInfoMapper,
|
|
CCBElectronicCashAccountInfoEntity.class);
|
|
Map<String, List<CCBElectronicCashAccountInfoEntity>> groupByCardHolderName = infoList.stream().filter(item ->StrUtil.isNotEmpty(item.getCardHolderName())).collect(Collectors.groupingBy(CCBElectronicCashAccountInfoEntity::getCardHolderName));
|
|
|
|
Set<String> uniqueKeySet = new HashSet();
|
|
List<PlateNumberInfo> plateNumberInfoList = new ArrayList<>();
|
|
|
|
for (CCBElectronicCashBankStatementEntity entity : entityList) {
|
|
String sourceFile = entity.getSourceFile();
|
|
try {
|
|
String cardHolderName = entity.getCardHolderName();
|
|
if (StringUtils.isEmpty(cardHolderName)) {
|
|
continue;
|
|
}
|
|
|
|
// 流水
|
|
BankStatement bs = new BankStatement();
|
|
bs.setBankName(BANK_NAME);
|
|
bs.setCardHolderName(cardHolderName);
|
|
|
|
|
|
|
|
if(StrUtil.isNotEmpty(cardHolderName) && StrUtil.isEmpty(bs.getIdCardNo())) {
|
|
if (groupByCardHolderName.containsKey(cardHolderName)) {
|
|
List<CCBElectronicCashAccountInfoEntity> infoEntities = groupByCardHolderName.get(cardHolderName);
|
|
for (CCBElectronicCashAccountInfoEntity infoEntity : infoEntities) {
|
|
if (StrUtil.isNotEmpty(infoEntity.getIdCardNo())) {
|
|
bs.setIdCardNo(infoEntity.getIdCardNo());
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
|
|
bs.setCardNumber(entity.getCardNumber());
|
|
// 1 货币类型
|
|
// 人民币
|
|
bs.setTransCurrencyType(Constants.CURRENCY_TYPE_CHINA);
|
|
|
|
// 2 交易金额 可能是double或者int类型
|
|
String expenditureAmount = entity.getExpenditureAmount();
|
|
if (StrUtil.isNotEmpty(expenditureAmount)) {
|
|
bs.setTransactionAmount(new BigDecimal(expenditureAmount));
|
|
}
|
|
|
|
String incomeAmount = entity.getIncomeAmount();
|
|
if (StrUtil.isNotEmpty(incomeAmount)) {
|
|
BigDecimal value =new BigDecimal(incomeAmount);
|
|
bs.setTransactionAmount(BigDecimal.ZERO.subtract(value));
|
|
}
|
|
|
|
// 3 余额
|
|
bs.setBalance(BigDecimal.valueOf(Double.parseDouble(entity.getBalance())));
|
|
// 4 交易日期
|
|
String transDate = entity.getTransDate();
|
|
if (StrUtil.isNotEmpty(transDate)) {
|
|
String transTime = entity.getTransTime();
|
|
try {
|
|
if (StrUtil.isNotEmpty(transTime)) {
|
|
bs.setTransactionTime(DateUtil.parse(transDate + " " + transTime, "yyyy-MM-dd HH:mm:ss"));
|
|
} else {
|
|
bs.setTransactionTime(DateUtil.parse(transDate, "yyyy-MM-dd"));
|
|
}
|
|
} catch (Exception e) {
|
|
log.error("解析日期出错", e);
|
|
|
|
|
|
throw new AnalyzeDataFailedException(
|
|
StrUtil.format("日期格式为：yyyy-MM-dd HH:mm:ss，日期为：{}", transDate), e, sourceFile);
|
|
}
|
|
}
|
|
|
|
String counterpartyName = entity.getCounterpartyName();
|
|
bs.setCounterpartyName(counterpartyName);
|
|
|
|
bs.setCounterpartyAccount(entity.getCounterpartyAccount());
|
|
bs.setCounterpartyBankName(entity.getCounterpartyBankName());
|
|
bs.setTransRemark(entity.getTansRemark());
|
|
|
|
String md5Id = HelperUtil.generateMD5Id(bs,caseId);
|
|
//未导入数据内部去重
|
|
if(HelperUtil.deduplication(md5Id,uniqueKeySet)){
|
|
continue;
|
|
}
|
|
|
|
bs.setSourceFile(entity.getSourceFile());
|
|
bs.setId(md5Id);
|
|
bs.setCaseId(caseId);
|
|
try {
|
|
BeanUtils.beanAttributeValueTrim(bs);
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
}
|
|
bsList.add(bs);
|
|
HelperUtil.extractPlateNumber(bs,plateNumberInfoList);
|
|
|
|
if (bsList.size() >= Constants.BATCH_SIZE) {
|
|
List<BankStatement> dest = HelperUtil.getDest(bsList);
|
|
|
|
HelperUtil.batchInsert2Es(dest, caseId);
|
|
|
|
bsList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
}
|
|
} catch (Exception e) {
|
|
importResultService.record(caseId, BANK_NAME, e,sourceFile);
|
|
}
|
|
}
|
|
uniqueKeySet.clear();
|
|
HelperUtil.batchInsertPlateNumber(plateNumberInfoList);
|
|
// 保存数据库
|
|
if (!bsList.isEmpty()) {
|
|
List<BankStatement> dest = HelperUtil.getDest(bsList);
|
|
HelperUtil.batchInsert2Es(dest, caseId);
|
|
}
|
|
}
|
|
|
|
private void analyzeRegularBS(String caseId) {
|
|
|
|
List<BankStatement> bsList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
|
|
List<CCBRegularBankStatementEntity> entityList =
|
|
HelperUtil.getEntityList(esCCBRegularBankStatementMapper, CCBRegularBankStatementEntity.class);
|
|
// 根据名称去查找基本信息
|
|
List<CCBRegularAccountInfoEntity> infoList = HelperUtil.getEntityList(esCCBRegularAccountInfoMapper, CCBRegularAccountInfoEntity.class);
|
|
Map<String, List<CCBRegularAccountInfoEntity>> groupByCardHolderName = infoList.stream().filter(item ->StrUtil.isNotEmpty(item.getCardHolderName())).collect(Collectors.groupingBy(CCBRegularAccountInfoEntity::getCardHolderName));
|
|
|
|
|
|
Set<String> uniqueKeySet = new HashSet();
|
|
List<PlateNumberInfo> plateNumberInfoList = new ArrayList<>();
|
|
for (CCBRegularBankStatementEntity entity : entityList) {
|
|
String sourceFile = entity.getSourceFile();
|
|
try {
|
|
String cardHolderName = entity.getCardHolderName();
|
|
if (StringUtils.isEmpty(cardHolderName)) {
|
|
continue;
|
|
}
|
|
|
|
// 流水
|
|
BankStatement bs = new BankStatement();
|
|
bs.setBankName(BANK_NAME);
|
|
bs.setCardHolderName(cardHolderName);
|
|
|
|
if(StrUtil.isNotEmpty(cardHolderName) && StrUtil.isEmpty(bs.getIdCardNo())) {
|
|
if (groupByCardHolderName.containsKey(cardHolderName)) {
|
|
List<CCBRegularAccountInfoEntity> infoEntities = groupByCardHolderName.get(cardHolderName);
|
|
for (CCBRegularAccountInfoEntity infoEntity : infoEntities) {
|
|
if (StrUtil.isNotEmpty(infoEntity.getIdCardNo())) {
|
|
bs.setIdCardNo(infoEntity.getIdCardNo());
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
bs.setCardNumber(entity.getCardNumber());
|
|
// 1 货币类型
|
|
// 人民币
|
|
bs.setTransCurrencyType(Constants.CURRENCY_TYPE_CHINA);
|
|
|
|
// 2 交易金额 可能是double或者int类型
|
|
BigDecimal transactionAmount = entity.getTransactionAmount();
|
|
String loanFlag = entity.getLoanFlag();
|
|
if (StrUtil.isNotEmpty(loanFlag)) {
|
|
if (Objects.equals("贷", loanFlag)) {
|
|
bs.setTransactionAmount(transactionAmount);
|
|
} else {
|
|
bs.setTransactionAmount(BigDecimal.ZERO.subtract(transactionAmount));
|
|
}
|
|
}
|
|
|
|
// 3 余额
|
|
bs.setBalance(entity.getBalance());
|
|
// 4 交易日期
|
|
String transDate = entity.getTransDate();
|
|
if (StrUtil.isNotEmpty(transDate)) {
|
|
String transTime = entity.getTransTime();
|
|
try {
|
|
if (StrUtil.isNotEmpty(transTime)) {
|
|
bs.setTransactionTime(DateUtil.parse(transDate + " " + transTime, "yyyy-MM-dd HH:mm:ss"));
|
|
} else {
|
|
bs.setTransactionTime(DateUtil.parse(transDate, "yyyy-MM-dd"));
|
|
}
|
|
} catch (Exception e) {
|
|
log.error("日期转换异常", e);
|
|
|
|
|
|
throw new AnalyzeDataFailedException(StrUtil.format("日期转换异常，日期：{}，时间：{}", transDate), e, sourceFile);
|
|
}
|
|
}
|
|
|
|
String counterpartyName = entity.getCounterpartyName();
|
|
bs.setCounterpartyName(counterpartyName);
|
|
|
|
bs.setTransactionInstitutions(entity.getTransactionInstitutions());
|
|
bs.setCounterpartyAccount(entity.getCounterpartyAccount());
|
|
bs.setTransChannel(entity.getTransChannel());
|
|
bs.setSummary(entity.getSummary());
|
|
|
|
String md5Id = HelperUtil.generateMD5Id(bs,caseId);
|
|
//未导入数据内部去重
|
|
if(HelperUtil.deduplication(md5Id,uniqueKeySet)){
|
|
continue;
|
|
}
|
|
|
|
bs.setSourceFile(entity.getSourceFile());
|
|
bs.setId(md5Id);
|
|
bs.setCaseId(caseId);
|
|
try {
|
|
BeanUtils.beanAttributeValueTrim(bs);
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
}
|
|
bsList.add(bs);
|
|
HelperUtil.extractPlateNumber(bs,plateNumberInfoList);
|
|
|
|
|
|
if (bsList.size() >= Constants.BATCH_SIZE) {
|
|
List<BankStatement> dest = HelperUtil.getDest(bsList);
|
|
|
|
HelperUtil.batchInsert2Es(dest, caseId);
|
|
|
|
bsList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
}
|
|
} catch (Exception e) {
|
|
importResultService.record(caseId, BANK_NAME, e,sourceFile);
|
|
}
|
|
}
|
|
uniqueKeySet.clear();
|
|
HelperUtil.batchInsertPlateNumber(plateNumberInfoList);
|
|
// 保存数据库
|
|
if (!bsList.isEmpty()) {
|
|
List<BankStatement> dest = HelperUtil.getDest(bsList);
|
|
HelperUtil.batchInsert2Es(dest, caseId);
|
|
}
|
|
}
|
|
|
|
private void analyzeCurrentBS(String caseId) {
|
|
|
|
List<BankStatement> bsList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
|
|
List<CCBCurrentBankStatementEntity> entityList =
|
|
HelperUtil.getEntityList(esCCBCurrentBankStatementMapper, CCBCurrentBankStatementEntity.class);
|
|
List<CCBCurrentAccountInfoEntity> oaiList =
|
|
HelperUtil.getEntityList(accountInfoMapper, CCBCurrentAccountInfoEntity.class);
|
|
Map<String, List<CCBCurrentAccountInfoEntity>> groupByCardNumber = oaiList.stream().filter(item ->StrUtil.isNotEmpty(item.getCardNumber())).collect(Collectors.groupingBy(CCBCurrentAccountInfoEntity::getCardNumber));
|
|
Map<String, List<CCBCurrentAccountInfoEntity>> groupByCardHolderName = oaiList.stream().filter(item ->StrUtil.isNotEmpty(item.getCardHolderName())).collect(Collectors.groupingBy(CCBCurrentAccountInfoEntity::getCardHolderName));
|
|
|
|
|
|
Map<String, List<CCBCurrentBankStatementEntity>> map = new HashMap<>();
|
|
Set<String> uniqueKeySet = new HashSet();
|
|
List<PlateNumberInfo> plateNumberInfoList = new ArrayList<>();
|
|
for (CCBCurrentBankStatementEntity entity : entityList) {
|
|
String sourceFile = entity.getSourceFile();
|
|
|
|
try {
|
|
boolean isSame = false;
|
|
|
|
String cardHolderName = entity.getCardHolderName();
|
|
String cardNumber = entity.getCardNumber();
|
|
|
|
if (StringUtils.isEmpty(cardHolderName)) {//这个判断直接过滤了企业数据
|
|
if (groupByCardNumber.containsKey(cardNumber)) {
|
|
List<CCBCurrentAccountInfoEntity> tempList = groupByCardNumber.get(cardNumber);
|
|
for (CCBCurrentAccountInfoEntity oai : tempList) {
|
|
if(StrUtil.isNotEmpty(oai.getCardHolderName())){
|
|
cardHolderName = oai.getCardHolderName();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (HelperUtil.isContainsChinese(entity.getTransDate())) {
|
|
continue;
|
|
}
|
|
|
|
// 去重
|
|
String serialNumber = entity.getSerialNumber();
|
|
if (StringUtils.isEmpty(serialNumber)) {
|
|
serialNumber = "-";
|
|
}
|
|
|
|
List<CCBCurrentBankStatementEntity> value = map.getOrDefault(serialNumber, null);
|
|
if (value == null) {
|
|
value = new ArrayList<>();
|
|
value.add(entity);
|
|
|
|
map.put(serialNumber, value);
|
|
} else {
|
|
for (CCBCurrentBankStatementEntity entry : value) {
|
|
if (HelperUtil.isTheSameBankStatement(entity, entry)) {
|
|
isSame = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (isSame) {
|
|
continue;
|
|
} else {
|
|
map.get(serialNumber).add(entity);
|
|
}
|
|
}
|
|
|
|
// 流水
|
|
BankStatement bs = new BankStatement();
|
|
bs.setBankName(BANK_NAME);
|
|
bs.setCardHolderName(cardHolderName);
|
|
// 根据名称去查找基本信息
|
|
|
|
if(StrUtil.isNotEmpty(cardHolderName) && StrUtil.isEmpty(bs.getIdCardNo())) {
|
|
if (groupByCardHolderName.containsKey(cardHolderName)) {
|
|
List<CCBCurrentAccountInfoEntity> infoEntities = groupByCardHolderName.get(cardHolderName);
|
|
for (CCBCurrentAccountInfoEntity infoEntity : infoEntities) {
|
|
if (StrUtil.isNotEmpty(infoEntity.getIdCardNo())) {
|
|
bs.setIdCardNo(infoEntity.getIdCardNo());
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
bs.setCardNumber(entity.getCardNumber());
|
|
// 1 货币类型
|
|
// 人民币
|
|
bs.setTransCurrencyType(Constants.CURRENCY_TYPE_CHINA);
|
|
|
|
// 2 交易金额 可能是double或者int类型
|
|
try {
|
|
if(StrUtil.isEmpty(entity.getLoanFlag()) && entity.getExpenditureAmount() != null && entity.getIncomeAmount() != null){
|
|
BigDecimal expenditureAmount = BigDecimal.valueOf(Double.parseDouble(entity.getExpenditureAmount()));
|
|
BigDecimal incomeAmount = BigDecimal.valueOf(Double.parseDouble(entity.getIncomeAmount()));
|
|
bs.setTransactionAmount(incomeAmount.subtract(expenditureAmount));
|
|
}else {
|
|
BigDecimal transactionAmount = new BigDecimal(entity.getTransactionAmount());
|
|
if(entity.getLoanFlag().contains("借")){
|
|
bs.setTransactionAmount(transactionAmount.negate());
|
|
} else {
|
|
bs.setTransactionAmount(transactionAmount);
|
|
}
|
|
}
|
|
|
|
} catch (Exception e) {
|
|
throw new AnalyzeDataFailedException("金额解析异常", e, sourceFile);
|
|
}
|
|
|
|
// 3 余额
|
|
bs.setBalance(BigDecimal.valueOf(Double.parseDouble(entity.getBalance())));
|
|
// 4 交易日期
|
|
String transDate = entity.getTransDate();
|
|
if (StrUtil.isNotEmpty(transDate)) {
|
|
String transTime = entity.getTransTime();
|
|
try {
|
|
if (StrUtil.isNotEmpty(transTime)) {
|
|
bs.setTransactionTime(DateUtil.parse(transDate + " " + transTime, "yyyy-MM-dd HH:mm:ss"));
|
|
} else {
|
|
bs.setTransactionTime(DateUtil.parse(transDate, "yyyy-MM-dd"));
|
|
}
|
|
} catch (Exception e) {
|
|
log.error("日期转换异常", e);
|
|
throw new AnalyzeDataFailedException(StrUtil.format("日期转换异常:{}", transDate), e, sourceFile);
|
|
}
|
|
}else {
|
|
continue;
|
|
}
|
|
|
|
String counterpartyName = entity.getCounterpartyName();
|
|
bs.setCounterpartyName(counterpartyName);
|
|
|
|
bs.setTransactionInstitutions(entity.getTransactionInstitutions());
|
|
bs.setCounterpartyAccount(entity.getCounterpartyAccount());
|
|
bs.setCounterpartyBankName(entity.getCounterpartyBankName());
|
|
bs.setTransChannel(entity.getTransChannel());
|
|
bs.setTransRemark(entity.getTansRemark());
|
|
bs.setSummary(entity.getSummary());
|
|
|
|
String md5Id = HelperUtil.generateMD5Id(bs,caseId);
|
|
//未导入数据内部去重
|
|
if(HelperUtil.deduplication(md5Id,uniqueKeySet)){
|
|
continue;
|
|
}
|
|
|
|
bs.setSourceFile(entity.getSourceFile());
|
|
bs.setId(md5Id);
|
|
bs.setCaseId(caseId);
|
|
try {
|
|
BeanUtils.beanAttributeValueTrim(bs);
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
}
|
|
bsList.add(bs);
|
|
HelperUtil.extractPlateNumber(bs,plateNumberInfoList);
|
|
|
|
|
|
if (bsList.size() >= Constants.BATCH_SIZE) {
|
|
List<BankStatement> dest = HelperUtil.getDest(bsList);
|
|
|
|
HelperUtil.batchInsert2Es(dest, caseId);
|
|
|
|
bsList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
}
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
importResultService.record(caseId, BANK_NAME, e,sourceFile);
|
|
}
|
|
}
|
|
uniqueKeySet.clear();
|
|
HelperUtil.batchInsertPlateNumber(plateNumberInfoList);
|
|
// 保存数据库
|
|
if (!bsList.isEmpty()) {
|
|
List<BankStatement> dest = HelperUtil.getDest(bsList);
|
|
HelperUtil.batchInsert2Es(dest, caseId);
|
|
}
|
|
}
|
|
|
|
public void importData(File file,String caseId) throws Exception {
|
|
|
|
List<File> fileList = FileUtil.loopFiles(file);
|
|
|
|
if (fileList.isEmpty()) {
|
|
throw new ImportDataFailedException(
|
|
StrUtil.format("导入失败，未找到Excel文件:{}", file.getName()), file.getAbsolutePath());
|
|
}
|
|
|
|
// 剔除汇总表
|
|
fileList = fileList.stream()
|
|
.filter(f -> {
|
|
if (FileUtil.mainName(f).contains("汇总")) {
|
|
AnalyzeFileHelper.putBSFile(f);
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
})
|
|
.collect(Collectors.toList());
|
|
List<File> excelFileList = HelperUtil.getExcelFile(fileList);
|
|
for (File excelFile : excelFileList) {
|
|
try {
|
|
analysisCCBExcel(excelFile,caseId);
|
|
} catch (Exception e) {
|
|
importResultService.record(caseId, BANK_NAME, e);
|
|
}
|
|
}
|
|
}
|
|
|
|
private void analysisCCBExcel(File excelFile,String caseId) throws Exception {
|
|
ZipSecureFile.setMinInflateRatio(0.001);
|
|
String absolutePath = excelFile.getAbsolutePath();
|
|
Workbook wb = new Workbook(absolutePath);
|
|
WorksheetCollection worksheets = wb.getWorksheets();
|
|
String sourceFile = HelperUtil.getSourceFileName(absolutePath,BANK_NAME);
|
|
int count = worksheets.getCount();
|
|
for (int sheetNo = 0; sheetNo < count; sheetNo++) {
|
|
Worksheet ws = worksheets.get(sheetNo);
|
|
String sheetName = ws.getName();
|
|
Cells cells = ws.getCells();
|
|
String ssss = sourceFile + BankStatementConstants.NAME_WITH_SHEET_NAME + sheetName;
|
|
try {
|
|
if (AsposeUtil.getCell(cells, "有无查询结果") != null) {
|
|
continue;
|
|
}else if (AsposeUtil.getCell(cells, "查询无数据") != null) {
|
|
continue;
|
|
} else if (AsposeUtil.getCell(cells, "个人账户信息") != null) {
|
|
// 读取账户信息
|
|
readAccountInfo(excelFile, cells, sheetNo, ssss);
|
|
} else if (AsposeUtil.getCell(cells, "个人活期明细信息") != null) {
|
|
// 读取活期信息
|
|
readCurrentDetailsV2(excelFile, cells, sheetNo, ssss);
|
|
} else if (AsposeUtil.getCell(cells, "个人定期明细信息") != null) {
|
|
// 读取定期明细
|
|
readRegularDetailsV2(excelFile, cells, sheetNo, ssss);
|
|
} else if (AsposeUtil.getCell(cells, "新一代电子现金") != null || AsposeUtil.getCell(cells, "电子现金-新一代") != null) {
|
|
// 读取电子现金明细
|
|
readElectronicCashDetails(excelFile, cells, sheetNo, ssss);
|
|
} else if (AsposeUtil.getCell(cells, "企业账户信息") != null) {
|
|
// 读取企业账户信息
|
|
readCompanyAccountInfo(excelFile, cells, sheetNo, ssss);
|
|
} else if (AsposeUtil.getCell(cells, "企业活期明细信息") != null) {
|
|
// 读取活期明细
|
|
// readCompanyCurrentDetails(excelFile, cells, sheetNo);
|
|
xiangjiaoTemplate(excelFile, sheetNo, ssss);
|
|
} else if (AsposeUtil.getCell(cells, "企业定期明细信息") != null) {
|
|
// 读取定期明细
|
|
readCompanyRegularDetails(excelFile, cells, sheetNo);
|
|
} else if (AsposeUtil.getCell(cells, "笔号") != null) {
|
|
if(AsposeUtil.getCell(cells, "交易日期") != null ){
|
|
readBSTemplate20240417(absolutePath, sheetNo,caseId, ssss);
|
|
}else if(AsposeUtil.getCell(cells, "开户日期") != null ){
|
|
readOAITemplate20240417(absolutePath, sheetNo,caseId, ssss);
|
|
}
|
|
} else if (AsposeUtil.getCell(cells, "错误原因") != null) {
|
|
continue;
|
|
}else {
|
|
throw new TemplateNotFindException(ssss);
|
|
}
|
|
|
|
} catch (Exception e) {
|
|
importResultService.record(caseId, BANK_NAME, e,ssss);
|
|
}
|
|
}
|
|
}
|
|
|
|
private void readOAITemplate20240417(String excelFile, int sheetNo, String caseId, String sourceFile) {
|
|
ExcelReader reader = EasyExcel.read(excelFile).build();
|
|
ReadSheet sheet = EasyExcel.readSheet(sheetNo)
|
|
.headRowNumber(1)
|
|
.head(CCBCurrentAccountInfoEntity.class)
|
|
.registerReadListener(readTemplate20240417OAIReadListener(caseId,sourceFile))
|
|
.build();
|
|
reader.read(sheet);
|
|
}
|
|
|
|
private void readBSTemplate20240417(String excelFile,int sheetNo,String caseId,String sourceFile) {
|
|
ExcelReader reader = EasyExcel.read(excelFile).build();
|
|
ReadSheet sheet = EasyExcel.readSheet(sheetNo)
|
|
.headRowNumber(1)
|
|
.head(CCBCurrentBankStatementEntity.class)
|
|
.registerReadListener(readTemplate20240417ReadListener(caseId,sourceFile))
|
|
.build();
|
|
reader.read(sheet);
|
|
}
|
|
|
|
public ReadListener<CCBCurrentAccountInfoEntity> readTemplate20240417OAIReadListener(String caseId,String sourceFile) {
|
|
return new ReadListener<CCBCurrentAccountInfoEntity>() {
|
|
|
|
List<CCBCurrentAccountInfoEntity> cacheList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
|
|
@Override
|
|
public void invoke(CCBCurrentAccountInfoEntity entity, AnalysisContext context) {
|
|
entity.setId(IdUtil.objectId());
|
|
entity.setSourceFile(sourceFile);
|
|
entity.setCaseId(caseId);
|
|
cacheList.add(entity);
|
|
if (cacheList.size() >= Constants.BATCH_SIZE) {
|
|
save();
|
|
}
|
|
}
|
|
|
|
@Override
|
|
public void doAfterAllAnalysed(AnalysisContext context) {
|
|
if (!cacheList.isEmpty()) {
|
|
save();
|
|
}
|
|
}
|
|
|
|
private void save() {
|
|
accountInfoMapper.insertBatch(cacheList);
|
|
cacheList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
}
|
|
};
|
|
}
|
|
|
|
public ReadListener<CCBCurrentBankStatementEntity> readTemplate20240417ReadListener(String caseId,String sourceFile) {
|
|
return new ReadListener<CCBCurrentBankStatementEntity>() {
|
|
|
|
List<CCBCurrentBankStatementEntity> cacheList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
|
|
@Override
|
|
public void invoke(CCBCurrentBankStatementEntity entity, AnalysisContext context) {
|
|
entity.setId(IdUtil.objectId());
|
|
entity.setSourceFile(sourceFile);
|
|
entity.setCaseId(caseId);
|
|
cacheList.add(entity);
|
|
if (cacheList.size() >= Constants.BATCH_SIZE) {
|
|
save();
|
|
}
|
|
}
|
|
|
|
@Override
|
|
public void doAfterAllAnalysed(AnalysisContext context) {
|
|
if (!cacheList.isEmpty()) {
|
|
save();
|
|
}
|
|
}
|
|
|
|
private void save() {
|
|
esCCBCurrentBankStatementMapper.insertBatch(cacheList);
|
|
cacheList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
}
|
|
};
|
|
}
|
|
|
|
private void readCompanyRegularDetails(File excelFile, Cells cells, int sheetNo) {
|
|
Class<CCBRegularBankStatementEntity> headClazz = CCBRegularBankStatementEntity.class;
|
|
String sourceFile = HelperUtil.getSourceFileName(excelFile.getAbsolutePath(),BANK_NAME);
|
|
Cell startCell = null;
|
|
Cell endCell = null;
|
|
int startRow = 0;
|
|
int endRow = 0;
|
|
|
|
for (; ; ) {
|
|
startCell = AsposeUtil.getCell(cells, "交易日期", startRow);
|
|
if (startCell == null) {
|
|
break;
|
|
}
|
|
|
|
startRow = startCell.getRow() + 1; // headRowNum
|
|
endCell = AsposeUtil.getCell(cells, "根据查询条件", startRow);
|
|
if (endCell == null) {
|
|
// 设置最后一行
|
|
endRow = cells.getMaxRow() + 1;
|
|
} else {
|
|
endRow = endCell.getRow() + 1;
|
|
}
|
|
|
|
try (ExcelReader r = EasyExcel.read(excelFile).build()) {
|
|
ReadSheet rs = EasyExcel.readSheet(sheetNo)
|
|
.head(headClazz)
|
|
.headRowNumber(startRow)
|
|
.registerReadListener(HelperUtil.getReadListener(
|
|
esCCBRegularBankStatementMapper, headClazz, startRow, endRow,sourceFile))
|
|
.build();
|
|
r.read(rs);
|
|
|
|
|
|
} catch (Exception e) {
|
|
|
|
// if (e instanceof ExcelAnalysisStopException) {
|
|
// continue;
|
|
// }
|
|
//
|
|
// log.error("读取企业定期流水信息失败: " + e.getMessage(), e);
|
|
// throw new ImportDataFailedException("读取企业定期流水信息失败: " + e.getMessage(), excelFile.getAbsolutePath());
|
|
}
|
|
|
|
startRow = endRow;
|
|
}
|
|
}
|
|
|
|
// 读取活期明细
|
|
private void readCompanyCurrentDetails(File excelFile, Cells cells, int sheetNo) {
|
|
String sourceFile = HelperUtil.getSourceFileName(excelFile.getAbsolutePath(),BANK_NAME);
|
|
Class<CCBCurrentBankStatementEntity> headClazz = CCBCurrentBankStatementEntity.class;
|
|
|
|
Cell startCell = null;
|
|
Cell endCell = null;
|
|
int startRow = 0;
|
|
int endRow = 0;
|
|
|
|
for (; ; ) {
|
|
startCell = AsposeUtil.getCell(cells, "交易日期", startRow);
|
|
if (startCell == null) {
|
|
break;
|
|
}
|
|
|
|
startRow = startCell.getRow() + 1; // headRowNum
|
|
endCell = AsposeUtil.getCell(cells, "根据查询条件", startRow);
|
|
if (endCell == null) {
|
|
// 设置最后一行
|
|
endRow = cells.getMaxRow() + 1;
|
|
} else {
|
|
endRow = endCell.getRow() + 1;
|
|
}
|
|
|
|
try (ExcelReader r = EasyExcel.read(excelFile).build()) {
|
|
ReadSheet rs = EasyExcel.readSheet(sheetNo)
|
|
.head(headClazz)
|
|
.headRowNumber(startRow)
|
|
.registerReadListener(HelperUtil.getReadListener(
|
|
esCCBCurrentBankStatementMapper, headClazz, startRow, endRow,sourceFile))
|
|
.build();
|
|
r.read(rs);
|
|
|
|
|
|
} catch (Exception e) {
|
|
|
|
if (e instanceof ExcelAnalysisStopException) {
|
|
continue;
|
|
}
|
|
|
|
log.error("读取企业活期流水信息失败: " + e.getMessage(), e);
|
|
throw new ImportDataFailedException("读取企业活期流水信息失败: " + e.getMessage(), sourceFile);
|
|
}
|
|
|
|
startRow = endRow;
|
|
}
|
|
}
|
|
|
|
private void xiangjiaoTemplate(File excelFile, int sheetNo , String sourceFile) throws Exception {
|
|
List<AnalyzeItem> itemList = getAnalyzeItemList(excelFile.getAbsolutePath(), sheetNo);
|
|
|
|
for (AnalyzeItem item : itemList) {
|
|
|
|
try (ExcelReader r = EasyExcel.read(excelFile).build()) {
|
|
ReadSheet readSheet = EasyExcel.readSheet(sheetNo)
|
|
.headRowNumber(item.headRowNumber)
|
|
.head(CCBCurrentBankStatementEntity.class)
|
|
.registerReadListener(getCurrentDetailListener(item,sourceFile))
|
|
.build();
|
|
r.read(readSheet);
|
|
|
|
} catch (Exception e) {
|
|
if (e instanceof ExcelAnalysisStopException) {
|
|
continue;
|
|
}
|
|
|
|
// 停止继续读取需要抛出异常，这里不必处理
|
|
log.error("读取个人活期明细信息失败: " + e.getMessage(), e);
|
|
throw new AnalyzeDataFailedException("读取个人活期明细信息失败: " + e.getMessage(), e);
|
|
}
|
|
}
|
|
}
|
|
|
|
private void readCompanyAccountInfo(File excelFile, Cells cells, int sheetNo,String sourceFile) {
|
|
|
|
// 关键字：企业活期账户信息
|
|
Cell startCell = null;
|
|
Cell endCell = null;
|
|
int startRow = 0;
|
|
int endRow = 0;
|
|
|
|
Class<CCBCurrentAccountInfoEntity> headClazz = CCBCurrentAccountInfoEntity.class;
|
|
|
|
String[] keys = {"企业活期账户信息", "企业定期账户信息"};
|
|
|
|
for (String key : keys) {
|
|
|
|
startCell = AsposeUtil.getCell(cells, key, startRow);
|
|
if (startCell == null) {
|
|
break;
|
|
}
|
|
|
|
startRow = startCell.getRow();
|
|
|
|
|
|
|
|
Cell cell = AsposeUtil.getCell(cells, "开户日期", startRow );
|
|
if (cell == null) {
|
|
continue;
|
|
}
|
|
startRow = cell.getRow();
|
|
|
|
// if (endCell != null) {
|
|
// startRow = endCell.getRow();
|
|
// }
|
|
endCell = AsposeUtil.getCell(cells, "根据查询条件", startRow);
|
|
if (endCell == null) {
|
|
// 设置最后一行
|
|
endRow = cells.getMaxRow() + 1;
|
|
} else {
|
|
endRow = endCell.getRow() + 1;
|
|
}
|
|
|
|
int headRowNumber = cell.getRow() + 1;
|
|
|
|
try (ExcelReader r = EasyExcel.read(excelFile).build()) {
|
|
ReadSheet rs = EasyExcel.readSheet(sheetNo)
|
|
.head(headClazz)
|
|
.headRowNumber(headRowNumber)
|
|
.registerReadListener(
|
|
HelperUtil.getReadListener(accountInfoMapper, headClazz, startRow, endRow,sourceFile))
|
|
.build();
|
|
r.read(rs);
|
|
} catch (Exception e) {
|
|
|
|
if (e instanceof ExcelAnalysisStopException) {
|
|
continue;
|
|
}
|
|
|
|
log.error("读取企业活期账户信息失败: " + e.getMessage(), e);
|
|
throw new ImportDataFailedException("读取企业活期账户信息失败: " + e.getMessage(), sourceFile);
|
|
}
|
|
}
|
|
}
|
|
|
|
// 读取定期数据，分为个人和企业两种数据
|
|
private void readRegularDetailsV2(File excelFile, Cells cells, int sheetNo,String sourceFile) throws Exception {
|
|
|
|
Cell cell = AsposeUtil.getCell(excelFile, sheetNo, "企业");
|
|
if (cell != null) {
|
|
readRegularDetailsForCompany(excelFile, sheetNo,sourceFile);
|
|
} else {
|
|
readRegularDetailsForPerson(excelFile, sheetNo,sourceFile);
|
|
}
|
|
}
|
|
|
|
private void readRegularDetailsForCompany(File excelFile, int sheetNo,String sourceFile) throws Exception {
|
|
List<AnalyzeItem> itemList = getAnalyzeItemListForCompany(excelFile.getAbsolutePath(), sheetNo);
|
|
|
|
for (AnalyzeItem item : itemList) {
|
|
|
|
try (ExcelReader r = EasyExcel.read(excelFile).build()) {
|
|
ReadSheet readSheet = EasyExcel.readSheet(sheetNo)
|
|
.headRowNumber(item.headRowNumber)
|
|
.head(CCBRegularBankStatementEntity.class)
|
|
.registerReadListener(getRegularDetailListener(item,sourceFile))
|
|
.build();
|
|
r.read(readSheet);
|
|
|
|
} catch (Exception e) {
|
|
// 停止继续读取需要抛出异常，这里不必处理
|
|
if (e instanceof ExcelAnalysisStopException) {
|
|
continue;
|
|
}
|
|
|
|
log.error("读取公司定期流水文件失败: " + e.getMessage(), e);
|
|
throw new ImportDataFailedException("读取公司定期流水文件失败: " + e.getMessage(), excelFile.getAbsolutePath());
|
|
}
|
|
}
|
|
}
|
|
|
|
private void readRegularDetailsForPerson(File excelFile, int sheetNo,String sourceFile) throws Exception {
|
|
List<AnalyzeItem> itemList = getAnalyzeItemList(excelFile.getAbsolutePath(), sheetNo);
|
|
|
|
for (AnalyzeItem item : itemList) {
|
|
|
|
try (ExcelReader r = EasyExcel.read(excelFile).build()) {
|
|
ReadSheet readSheet = EasyExcel.readSheet(sheetNo)
|
|
.headRowNumber(item.headRowNumber)
|
|
.head(CCBRegularBankStatementEntity.class)
|
|
.registerReadListener(getRegularDetailListener(item,sourceFile))
|
|
.build();
|
|
r.read(readSheet);
|
|
|
|
|
|
} catch (Exception e) {
|
|
// 停止继续读取需要抛出异常，这里不必处理
|
|
log.error("读取_accounts_info文件失败", e);
|
|
throw new AnalyzeDataFailedException("读取_accounts_info文件失败", e);
|
|
}
|
|
}
|
|
}
|
|
|
|
private ReadListener<CCBRegularBankStatementEntity> getRegularDetailListener(AnalyzeItem item,String sourceFile) {
|
|
AtomicInteger cnt = new AtomicInteger(item.headRowNumber + 1);
|
|
return new ReadListener<CCBRegularBankStatementEntity>() {
|
|
|
|
private List<CCBRegularBankStatementEntity> docList =
|
|
ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
|
|
@Override
|
|
public void invoke(CCBRegularBankStatementEntity entity, AnalysisContext context) {
|
|
|
|
String id = IdUtil.objectId();
|
|
|
|
|
|
entity.setId(id);
|
|
entity.setCardNumber(item.getCardNumber());
|
|
entity.setCardHolderName(item.getCardHolderName());
|
|
entity.setSourceFile(sourceFile);
|
|
docList.add(entity);
|
|
if (docList.size() >= Constants.BATCH_SIZE) {
|
|
saveToMongo();
|
|
docList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
}
|
|
|
|
if (cnt.incrementAndGet() > item.endRowNumber) {
|
|
saveToMongo();
|
|
throw new ExcelAnalysisStopException("stop running.");
|
|
}
|
|
}
|
|
|
|
private void saveToMongo() {
|
|
if (!docList.isEmpty()) {
|
|
esCCBRegularBankStatementMapper.insertBatch(docList);
|
|
}
|
|
}
|
|
|
|
@Override
|
|
public void doAfterAllAnalysed(AnalysisContext context) {
|
|
if (!docList.isEmpty()) {
|
|
saveToMongo();
|
|
}
|
|
}
|
|
};
|
|
}
|
|
|
|
// 个人活期明细信息
|
|
private void readCurrentDetailsV2(File excelFile, Cells cells, int sheetNo, String sourceFile) throws Exception {
|
|
List<AnalyzeItem> itemList = getAnalyzeItemList(excelFile.getAbsolutePath(), sheetNo);
|
|
|
|
for (AnalyzeItem item : itemList) {
|
|
|
|
try (ExcelReader r = EasyExcel.read(excelFile).build()) {
|
|
ReadSheet readSheet = EasyExcel.readSheet(sheetNo)
|
|
.headRowNumber(item.headRowNumber)
|
|
.head(CCBCurrentBankStatementEntity.class)
|
|
.registerReadListener(getCurrentDetailListener(item,sourceFile))
|
|
.build();
|
|
r.read(readSheet);
|
|
|
|
} catch (Exception e) {
|
|
if (e instanceof ExcelAnalysisStopException) {
|
|
continue;
|
|
}
|
|
|
|
// 停止继续读取需要抛出异常，这里不必处理
|
|
log.error("读取个人活期明细信息失败: " + e.getMessage(), e);
|
|
throw new AnalyzeDataFailedException("读取个人活期明细信息失败: " + e.getMessage(), e);
|
|
}
|
|
}
|
|
}
|
|
|
|
private ReadListener<CCBCurrentBankStatementEntity> getCurrentDetailListener(AnalyzeItem item,String sourceFile) {
|
|
AtomicInteger cnt = new AtomicInteger(item.headRowNumber + 1);
|
|
return new ReadListener<CCBCurrentBankStatementEntity>() {
|
|
|
|
private List<CCBCurrentBankStatementEntity> docList =
|
|
ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
|
|
@Override
|
|
public void invoke(CCBCurrentBankStatementEntity entity, AnalysisContext context) {
|
|
|
|
if (HelperUtil.isContainsChinese(entity.getId())) {
|
|
return;
|
|
}
|
|
|
|
String id = IdUtil.objectId();
|
|
|
|
|
|
entity.setId(id);
|
|
if(StrUtil.isEmpty(entity.getCardNumber())){
|
|
entity.setCardNumber(item.getCardNumber());
|
|
}
|
|
entity.setSourceFile(sourceFile);
|
|
entity.setCardHolderName(item.getCardHolderName());
|
|
|
|
docList.add(entity);
|
|
if (docList.size() >= Constants.BATCH_SIZE) {
|
|
saveToMongo();
|
|
docList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
}
|
|
|
|
if (cnt.incrementAndGet() > item.endRowNumber) {
|
|
saveToMongo();
|
|
throw new ExcelAnalysisStopException("stop running.");
|
|
}
|
|
}
|
|
|
|
private void saveToMongo() {
|
|
if (!docList.isEmpty()) {
|
|
esCCBCurrentBankStatementMapper.insertBatch(docList);
|
|
}
|
|
}
|
|
|
|
@Override
|
|
public void doAfterAllAnalysed(AnalysisContext context) {
|
|
if (!docList.isEmpty()) {
|
|
saveToMongo();
|
|
}
|
|
}
|
|
};
|
|
}
|
|
|
|
/**
|
|
* 读取电子现金明细
|
|
*/
|
|
private void readElectronicCashDetails(File excelFile, Cells cells, int sheetNo,String sourceFile) throws Exception {
|
|
List<AnalyzeItem> itemList = getAnalyzeItemList(excelFile.getAbsolutePath(), sheetNo);
|
|
|
|
for (AnalyzeItem item : itemList) {
|
|
|
|
try (ExcelReader r = EasyExcel.read(excelFile).build()) {
|
|
ReadSheet readSheet = EasyExcel.readSheet(sheetNo)
|
|
.headRowNumber(item.headRowNumber)
|
|
.head(CCBElectronicCashBankStatementEntity.class)
|
|
.registerReadListener(getElectronicCashDetailListener(item,sourceFile))
|
|
.build();
|
|
r.read(readSheet);
|
|
|
|
} catch (Exception e) {
|
|
|
|
if (e instanceof ExcelAnalysisStopException) {
|
|
// 停止继续读取需要抛出异常，这里不必处理
|
|
continue;
|
|
} else {
|
|
// 打印日志，抛出异常
|
|
log.error("读取电子现金明细异常", e);
|
|
throw new ImportDataFailedException(e.getMessage(), excelFile.getAbsolutePath());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private ReadListener<CCBElectronicCashBankStatementEntity> getElectronicCashDetailListener(AnalyzeItem item,String sourceFile) {
|
|
AtomicInteger cnt = new AtomicInteger(item.headRowNumber + 1);
|
|
return new ReadListener<CCBElectronicCashBankStatementEntity>() {
|
|
|
|
private List<CCBElectronicCashBankStatementEntity> docList =
|
|
ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
|
|
@Override
|
|
public void invoke(CCBElectronicCashBankStatementEntity entity, AnalysisContext context) {
|
|
String id = IdUtil.objectId();
|
|
|
|
entity.setId(id);
|
|
entity.setCardNumber(item.getCardNumber());
|
|
entity.setCardHolderName(item.getCardHolderName());
|
|
entity.setSourceFile(sourceFile);
|
|
docList.add(entity);
|
|
if (docList.size() >= Constants.BATCH_SIZE) {
|
|
saveToMongo();
|
|
docList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
}
|
|
|
|
if (cnt.incrementAndGet() > item.endRowNumber) {
|
|
saveToMongo();
|
|
throw new ExcelAnalysisStopException("stop running.");
|
|
}
|
|
}
|
|
|
|
private void saveToMongo() {
|
|
if (!docList.isEmpty()) {
|
|
esCCBElectronicCashBankStatementMapper.insertBatch(docList);
|
|
}
|
|
}
|
|
|
|
@Override
|
|
public void doAfterAllAnalysed(AnalysisContext context) {
|
|
if (!docList.isEmpty()) {
|
|
saveToMongo();
|
|
}
|
|
}
|
|
|
|
@Override
|
|
public void onException(Exception exception, AnalysisContext context) throws Exception {
|
|
HelperUtil.handleException(exception);
|
|
|
|
ReadListener.super.onException(exception, context);
|
|
}
|
|
};
|
|
}
|
|
|
|
/**
|
|
* 读取账户信息
|
|
*/
|
|
private void readAccountInfo(File excelFile, Cells cells, int sheetNo,String sourceFile) throws Exception {
|
|
// 三种账户信息
|
|
int currentStartRow = AsposeUtil.getRowNumber(excelFile, sheetNo, "活期账户信息", 0);
|
|
if (currentStartRow != -1) {
|
|
currentStartRow += 3;
|
|
// 根据查询条件
|
|
int currentEndRow = AsposeUtil.getRowNumber(excelFile, sheetNo, "根据查询条件", currentStartRow);
|
|
if (currentEndRow != -1) {
|
|
// 1、个人活期账户信息 个人活期账户信息
|
|
readCurrentAccountInfo(excelFile, sheetNo, currentStartRow, currentEndRow,sourceFile);
|
|
}
|
|
}
|
|
int regularStartRow = AsposeUtil.getRowNumber(excelFile, sheetNo, "定期账户信息", 0);
|
|
if (regularStartRow != -1) {
|
|
regularStartRow += 3;
|
|
int regularEndRow = AsposeUtil.getRowNumber(excelFile, sheetNo, "根据查询条件", regularStartRow);
|
|
if (regularEndRow != -1) {
|
|
// 2、个人定期账户信息
|
|
readRegularAccountInfo(excelFile, sheetNo, regularStartRow, regularEndRow,sourceFile);
|
|
}
|
|
}
|
|
|
|
int electronicCashStartRow = AsposeUtil.getRowNumber(excelFile, sheetNo, "个人电子现金账户信息", 0);
|
|
if (electronicCashStartRow != -1) {
|
|
electronicCashStartRow += 3;
|
|
int electronicCashEndRow =
|
|
AsposeUtil.getRowNumber(excelFile, sheetNo, "个人电子现金账户信息", electronicCashStartRow);
|
|
if (electronicCashEndRow != -1) {
|
|
// 3、个人电子现金账户信息
|
|
readElectronicCashAccountInfo(excelFile, sheetNo, electronicCashStartRow, electronicCashEndRow,sourceFile);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* 读取电子现金账户信息
|
|
*/
|
|
private void readElectronicCashAccountInfo(File excelFile, int sheetNo, int startRow, int endRow,String sourceFile) {
|
|
Class<CCBElectronicCashAccountInfoEntity> clazz = CCBElectronicCashAccountInfoEntity.class;
|
|
try (ExcelReader reader = EasyExcel.read(excelFile).build()) {
|
|
ReadSheet readSheet = EasyExcel.readSheet(sheetNo)
|
|
.headRowNumber(startRow)
|
|
.head(clazz)
|
|
.registerReadListener(HelperUtil.getReadListener(
|
|
esCCBElectronicCashAccountInfoMapper, clazz, startRow, endRow,sourceFile))
|
|
.build();
|
|
reader.read(readSheet);
|
|
} catch (Exception e) {
|
|
|
|
if (e instanceof ExcelAnalysisStopException) {
|
|
return;
|
|
}
|
|
|
|
log.error("read current account info error", e);
|
|
throw new ImportDataFailedException(e.getMessage(), excelFile.getAbsolutePath());
|
|
}
|
|
}
|
|
|
|
/**
|
|
* 读取定期账户信息
|
|
*/
|
|
private void readRegularAccountInfo(File excelFile, int sheetNo, int startRow, int endRow,String sourceFile) {
|
|
try (ExcelReader reader = EasyExcel.read(excelFile).build()) {
|
|
ReadSheet readSheet = EasyExcel.readSheet(sheetNo)
|
|
.headRowNumber(startRow)
|
|
.head(CCBRegularAccountInfoEntity.class)
|
|
.registerReadListener(
|
|
HelperUtil.getReadListener(esCCBRegularAccountInfoMapper, CCBRegularAccountInfoEntity.class, startRow, endRow,sourceFile))
|
|
.build();
|
|
reader.read(readSheet);
|
|
} catch (Exception e) {
|
|
if (e instanceof ExcelAnalysisStopException) {
|
|
return;
|
|
}
|
|
|
|
log.error("read current account info error", e);
|
|
throw new ImportDataFailedException(e.getMessage(), excelFile.getAbsolutePath());
|
|
}
|
|
}
|
|
|
|
/**
|
|
* 读取活期账户信息
|
|
*/
|
|
private void readCurrentAccountInfo(File excelFile, int sheetNo, int startRow, int endRow,String sourceFile) {
|
|
try (ExcelReader reader = EasyExcel.read(excelFile).build()) {
|
|
ReadSheet readSheet = EasyExcel.readSheet(sheetNo)
|
|
.headRowNumber(startRow)
|
|
.head(CCBCurrentAccountInfoEntity.class)
|
|
.registerReadListener(new ReadCCBCurrentAccountInfoListener(startRow, endRow, accountInfoMapper,sourceFile))
|
|
.build();
|
|
reader.read(readSheet);
|
|
} catch (Exception e) {
|
|
|
|
if (e instanceof ExcelAnalysisStopException) {
|
|
return;
|
|
}
|
|
|
|
log.error("read current account info error", e);
|
|
throw new ImportDataFailedException(e.getMessage(), excelFile.getAbsolutePath());
|
|
}
|
|
}
|
|
|
|
/**
|
|
* 通过读取汇总结果，得到相关文件
|
|
*/
|
|
private Set<String> readSummary(File f) {
|
|
|
|
List<CCBSummaryEntry> summaryList = new ArrayList<>();
|
|
|
|
Set<String> fileNameSet = new HashSet<>();
|
|
|
|
try (ExcelReader reader = EasyExcel.read(f).build()) {
|
|
ReadSheet sheet = EasyExcel.readSheet()
|
|
.headRowNumber(2)
|
|
.head(CCBSummaryEntry.class)
|
|
.registerReadListener(new ReadListener<CCBSummaryEntry>() {
|
|
@Override
|
|
public void invoke(CCBSummaryEntry entry, AnalysisContext context) {
|
|
summaryList.add(entry);
|
|
}
|
|
|
|
@Override
|
|
public void doAfterAllAnalysed(AnalysisContext context) {
|
|
// 读取文件结束
|
|
for (CCBSummaryEntry entry : summaryList) {
|
|
String resultFileName = entry.getResultFileName();
|
|
if (!com.inscloudtech.common.utils.StringUtils.isEmpty(resultFileName)) {
|
|
int idx = resultFileName.indexOf("_");
|
|
if (idx != -1) {
|
|
String fileName = resultFileName.substring(idx + 1);
|
|
|
|
Integer resultCount = entry.getResultCount();
|
|
if (resultCount != 0) {
|
|
fileNameSet.add(fileName);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
})
|
|
.build();
|
|
// 读文件
|
|
reader.read(sheet);
|
|
}
|
|
|
|
return fileNameSet;
|
|
}
|
|
|
|
// ------------------------------------------------------------------------------------------------------------------
|
|
|
|
private List<AnalyzeItem> getAnalyzeItemListForCompany(String filename, int sheetNo) throws Exception {
|
|
Workbook wb = new Workbook(filename);
|
|
WorksheetCollection worksheets = wb.getWorksheets();
|
|
|
|
Worksheet ws = worksheets.get(sheetNo);
|
|
|
|
String mainKey = "根据查询条件";
|
|
final String key = "客户名称";
|
|
|
|
Cells cells = ws.getCells();
|
|
|
|
Cell roundStartCell = cells.find(mainKey, null);
|
|
|
|
List<AnalyzeItem> itemList = new ArrayList<>();
|
|
|
|
for (; ; ) {
|
|
|
|
if (roundStartCell == null) {
|
|
break;
|
|
}
|
|
|
|
Cell roundEndCell = cells.find(mainKey, roundStartCell);
|
|
|
|
Cell cell = cells.find(key, roundStartCell);
|
|
if (cell == null) {
|
|
break;
|
|
}
|
|
|
|
int roundEndRow;
|
|
if (roundEndCell == null) {
|
|
roundEndRow = cells.getMaxRow();
|
|
} else {
|
|
roundEndRow = roundEndCell.getRow();
|
|
}
|
|
|
|
int row = cell.getRow();
|
|
|
|
while (row < roundEndRow) {
|
|
AnalyzeItem ai = new AnalyzeItem();
|
|
// 说明这一轮有数据
|
|
// 客户名称:徐唯，客户编号:667190000103481964，客户账号:3860740150310007432，一本通账号或卡号:4340613860503177，存款种类:一户通，币别:人民币元，钞汇标志:钞，册号:0，笔号:1
|
|
String complexStr = cell.getStringValue(); // 解析客户名称和客户账号
|
|
if (StrUtil.isNotEmpty(complexStr)) {
|
|
String[] strArr = complexStr.split("，");
|
|
if (strArr.length > 3) {
|
|
String nameStr = strArr[0];
|
|
String[] nameArr = nameStr.split(":");
|
|
if (nameArr.length == 2) {
|
|
ai.setCardHolderName(nameArr[1]);
|
|
} else {
|
|
break;
|
|
}
|
|
|
|
String cardNumberStr = strArr[2];
|
|
String[] cardNumberArr = cardNumberStr.split(":");
|
|
if (cardNumberArr.length == 2) {
|
|
ai.setCardNumber(cardNumberArr[1]);
|
|
} else {
|
|
break;
|
|
}
|
|
}
|
|
} else {
|
|
break;
|
|
}
|
|
|
|
int headRowNumber = cell.getRow() + 2;
|
|
|
|
ai.setHeadRowNumber(headRowNumber);
|
|
|
|
cell = cells.find(key, cell);
|
|
if (cell == null) {
|
|
ai.setEndRowNumber(roundEndRow);
|
|
itemList.add(ai);
|
|
|
|
break;
|
|
}
|
|
|
|
row = cell.getRow();
|
|
ai.setEndRowNumber(Math.min(row, roundEndRow));
|
|
|
|
itemList.add(ai);
|
|
}
|
|
|
|
// 这一轮没有数据，开始下一轮
|
|
roundStartCell = roundEndCell;
|
|
}
|
|
return itemList;
|
|
}
|
|
|
|
private List<AnalyzeItem> getAnalyzeItemList(String filename, int sheetNo) throws Exception {
|
|
Workbook wb = new Workbook(filename);
|
|
WorksheetCollection worksheets = wb.getWorksheets();
|
|
|
|
Worksheet ws = worksheets.get(sheetNo);
|
|
|
|
String mainKey = "根据查询条件";
|
|
final String key = "客户名称";
|
|
|
|
Cells cells = ws.getCells();
|
|
|
|
Cell roundStartCell = cells.find(mainKey, null);
|
|
|
|
List<AnalyzeItem> itemList = new ArrayList<>();
|
|
|
|
for (; ; ) {
|
|
|
|
if (roundStartCell == null) {
|
|
break;
|
|
}
|
|
|
|
Cell roundEndCell = cells.find(mainKey, roundStartCell);
|
|
|
|
Cell cell = cells.find(key, roundStartCell);
|
|
if (cell == null) {
|
|
break;
|
|
}
|
|
|
|
int roundEndRow;
|
|
if (roundEndCell == null) {
|
|
roundEndRow = cells.getMaxRow() + 1;
|
|
} else {
|
|
roundEndRow = roundEndCell.getRow();
|
|
}
|
|
|
|
int row = cell.getRow();
|
|
|
|
while (row <= roundEndRow) {
|
|
AnalyzeItem ai = new AnalyzeItem();
|
|
// 说明这一轮有数据
|
|
// 客户名称:徐唯，客户编号:667190000103481964，客户账号:3860740150310007432，一本通账号或卡号:4340613860503177，存款种类:一户通，币别:人民币元，钞汇标志:钞，册号:0，笔号:1
|
|
String complexStr = cell.getStringValue(); // 解析客户名称和客户账号
|
|
if (StrUtil.isNotEmpty(complexStr)) {
|
|
String[] strArr = complexStr.split("，");
|
|
if (strArr.length > 3) {
|
|
String nameStr = strArr[0];
|
|
String[] nameArr = nameStr.split(":");
|
|
if (nameArr.length == 2) {
|
|
ai.setCardHolderName(nameArr[1]);
|
|
} else {
|
|
break;
|
|
}
|
|
|
|
String cardNumberStr = strArr[2];
|
|
String[] cardNumberArr = cardNumberStr.split(":");
|
|
if (cardNumberArr.length == 2) {
|
|
ai.setCardNumber(cardNumberArr[1]);
|
|
} else {
|
|
break;
|
|
}
|
|
}
|
|
} else {
|
|
break;
|
|
}
|
|
|
|
int headRowNumber = cell.getRow() + 2;
|
|
|
|
ai.setHeadRowNumber(headRowNumber);
|
|
|
|
cell = cells.find(key, cell);
|
|
if (cell == null) {
|
|
ai.setEndRowNumber(roundEndRow);
|
|
itemList.add(ai);
|
|
|
|
break;
|
|
}
|
|
|
|
row = cell.getRow();
|
|
ai.setEndRowNumber(Math.min(row, roundEndRow));
|
|
|
|
itemList.add(ai);
|
|
}
|
|
|
|
// 这一轮没有数据，开始下一轮
|
|
roundStartCell = roundEndCell;
|
|
}
|
|
return itemList;
|
|
}
|
|
|
|
@Data
|
|
public static class AnalyzeItem {
|
|
private int headRowNumber;
|
|
private int endRowNumber;
|
|
private String cardHolderName;
|
|
private String cardNumber;
|
|
}
|
|
}
|