You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1152 lines
52 KiB
1152 lines
52 KiB
package com.inscloudtech.bankStatementAnalysis.helper;
|
|
|
|
import cn.hutool.core.collection.CollectionUtil;
|
|
import cn.hutool.core.date.DateUtil;
|
|
import cn.hutool.core.io.FileUtil;
|
|
import cn.hutool.core.util.IdUtil;
|
|
import cn.hutool.core.util.StrUtil;
|
|
import com.alibaba.excel.EasyExcel;
|
|
import com.alibaba.excel.ExcelReader;
|
|
import com.alibaba.excel.context.AnalysisContext;
|
|
import com.alibaba.excel.exception.ExcelAnalysisStopException;
|
|
import com.alibaba.excel.read.listener.ReadListener;
|
|
import com.alibaba.excel.read.metadata.ReadSheet;
|
|
import com.alibaba.excel.util.ListUtils;
|
|
import com.aspose.cells.Cell;
|
|
import com.aspose.cells.Cells;
|
|
import com.aspose.cells.Workbook;
|
|
import com.aspose.cells.Worksheet;
|
|
import com.baomidou.mybatisplus.core.toolkit.StringUtils;
|
|
|
|
import com.inscloudtech.bankStatementAnalysis.excel.CellsWrapper;
|
|
import com.inscloudtech.bankStatementAnalysis.helper.HelperUtil;
|
|
import com.inscloudtech.bankStatementAnalysis.mapper.*;
|
|
import com.inscloudtech.bankStatementAnalysis.service.ImportService;
|
|
import com.inscloudtech.common.constant.BankStatementConstants;
|
|
import com.inscloudtech.common.constant.Constants;
|
|
import com.inscloudtech.common.exception.dc.AnalyzeDataFailedException;
|
|
import com.inscloudtech.common.exception.dc.ImportDataFailedException;
|
|
import com.inscloudtech.common.exception.dc.TemplateNotFindException;
|
|
import com.inscloudtech.common.utils.bean.BeanUtils;
|
|
import com.inscloudtech.common.utils.file.FileUtils;
|
|
import com.inscloudtech.datacenter.domain.PlateNumberInfo;
|
|
import com.inscloudtech.datacenter.mapper.es.*;
|
|
import com.inscloudtech.datacenter.service.ImportResultService;
|
|
import com.inscloudtech.datacenter.domain.BankStatement;
|
|
import com.inscloudtech.datacenter.domain.OpeningAccountInfo;
|
|
|
|
import com.inscloudtech.bankStatementAnalysis.domain.entity.ABCCompanyInfoEntity;
|
|
import com.inscloudtech.bankStatementAnalysis.domain.entity.ABCCompanyStatementEntity;
|
|
import com.inscloudtech.bankStatementAnalysis.domain.entity.ABCCustomerInfoEntity;
|
|
import com.inscloudtech.bankStatementAnalysis.domain.entity.ABCCustomerStatementEntity;
|
|
|
|
import com.inscloudtech.bankStatementAnalysis.util.AsposeUtil;
|
|
|
|
import lombok.RequiredArgsConstructor;
|
|
import lombok.extern.slf4j.Slf4j;
|
|
import org.apache.poi.openxml4j.exceptions.OLE2NotOfficeXmlFileException;
|
|
import org.springframework.stereotype.Component;
|
|
|
|
import java.io.File;
|
|
import java.util.*;
|
|
import java.util.regex.Pattern;
|
|
import java.util.stream.Collectors;
|
|
|
|
/**
|
|
* 农业银行数据分析
|
|
*/
|
|
@Slf4j
|
|
@RequiredArgsConstructor
|
|
@Component
|
|
public class ABCDataAnalysisHelper {
|
|
|
|
private final ImportResultService importResultService;
|
|
private final ImportService importService;
|
|
|
|
private final EsABCCustomerStatementMapper esABCCustomerStatementMapper;
|
|
private final EsABCCompanyStatementMapper esABCCompanyStatementMapper;
|
|
private final EsABCCustomerInfoMapper esABCCustomerInfoMapper;
|
|
private final EsABCCompanyInfoMapper esABCCompanyInfoMapper;
|
|
private final ESOpeningAccountInfoMapper esOpeningAccountInfoMapper;
|
|
private final static String BANK_NAME = "农业银行";
|
|
private final static String IGNORE_SHEET_NAME = "金融资产列表,首页,子账户列表,冻结历史列表,共有权列表";
|
|
|
|
public void importData(File file, String caseId) {
|
|
List<File> fileList = FileUtil.loopFiles(file);
|
|
|
|
// 筛选文件
|
|
List<File> excelFileList = HelperUtil.getExcelFile(fileList);
|
|
|
|
//农行数据大致分为两类：
|
|
// 1. 开户信息和流水在同一个工作簿
|
|
// 2. 开户信息和流水分开工作簿
|
|
//农行的数据文件分两类：1、无法读取；2、可以读取
|
|
for (File excelFile : excelFileList) {
|
|
try {
|
|
String excelFileName = excelFile.getAbsolutePath();
|
|
String sourceFile = HelperUtil.getSourceFileName(excelFileName, BANK_NAME);
|
|
Workbook wb = new Workbook(excelFileName);
|
|
List<ABCCompanyInfoEntity> companyOAIList = new ArrayList<>();
|
|
boolean bsHasNotCard = false;
|
|
|
|
int count = wb.getWorksheets().getCount();
|
|
for (int sheetNo = 0; sheetNo < count; sheetNo++) {
|
|
try {
|
|
boolean templateIsExist = false;
|
|
Worksheet worksheet = wb.getWorksheets().get(sheetNo);
|
|
Cells cells = worksheet.getCells();
|
|
String sheetName = worksheet.getName();
|
|
String nameWithSheetName = sourceFile + BankStatementConstants.NAME_WITH_SHEET_NAME + sheetName;
|
|
Cell oaiCell1 = AsposeUtil.getCell(cells, "开户日期");
|
|
if (oaiCell1 != null) {//个人开户信息
|
|
if (excelFileName.endsWith(".xlsx")) {
|
|
personOai(oaiCell1, excelFileName, sheetNo, cells);
|
|
}
|
|
|
|
if (excelFileName.endsWith(".xls")) {
|
|
Cell tradeDateCell = AsposeUtil.getCell(cells, "发生额");
|
|
if (tradeDateCell != null) {//20240428新模板 个人，开户流水在一个模板里
|
|
readBsAndOai(excelFileName, cells, sheetNo, nameWithSheetName);
|
|
} else {
|
|
readOAI(cells);
|
|
}
|
|
}
|
|
|
|
templateIsExist = true;
|
|
}
|
|
|
|
if (!templateIsExist) {
|
|
Cell oaiCell2 = AsposeUtil.getCell(cells, "账户启用日期");
|
|
Cell oaiCell3 = AsposeUtil.getCell(cells, "合约建立日期");
|
|
|
|
if (oaiCell2 != null || oaiCell3 != null) {//公司开户信息
|
|
Cell tradeDateCell = AsposeUtil.getCell(cells, "交易日期");
|
|
if (tradeDateCell != null) {//20240416新模板 对公开户，流水在一个模板里
|
|
readBsAndOai(excelFileName, cells, sheetNo, nameWithSheetName);
|
|
templateIsExist = true;
|
|
} else {
|
|
Cell tempCell = oaiCell3;
|
|
if (oaiCell2 != null) {
|
|
tempCell = oaiCell2;
|
|
bsHasNotCard = true;//这种模板 开户+流水，但是流水里没有卡号，开户表里只能有一行
|
|
}
|
|
if (excelFileName.endsWith(".xls")) {
|
|
companyOAIList = readCompanyOAIByXls(cells);
|
|
} else {
|
|
companyOai(tempCell, excelFileName, sheetNo, companyOAIList);
|
|
}
|
|
templateIsExist = true;
|
|
}
|
|
|
|
}
|
|
}
|
|
|
|
if (!templateIsExist && bsHasNotCard) {
|
|
Cell bsCell1 = AsposeUtil.getCell(cells, "交易行省市代码");
|
|
if (bsCell1 != null) {//这种模板 开户+流水，但是流水里没有卡号
|
|
//2023081615505557832-账号：24055401040001038
|
|
if (excelFileName.endsWith(".xls")) {
|
|
readCompanyBSByXls(excelFileName, cells, companyOAIList.get(0));
|
|
} else {
|
|
companyBs(bsCell1, excelFileName, sheetNo, companyOAIList.get(0));//这个表的时间长度较长
|
|
}
|
|
templateIsExist = true;
|
|
}
|
|
}
|
|
|
|
if (!templateIsExist) {
|
|
Cell bsCell2 = AsposeUtil.getCell(cells, "合约账户余额");
|
|
if (bsCell2 != null) {
|
|
companyBs(bsCell2, excelFileName, sheetNo, null);
|
|
templateIsExist = true;
|
|
}
|
|
}
|
|
|
|
if (!templateIsExist) {
|
|
Cell bsCell3 = AsposeUtil.getCell(cells, "子账号序号");
|
|
if (bsCell3 != null) {
|
|
Cell temp = AsposeUtil.getCell(cells, "交易日期");
|
|
if (temp != null) {
|
|
personBs(bsCell3, excelFileName, sheetNo,nameWithSheetName);
|
|
templateIsExist = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!templateIsExist) {
|
|
Cell bsCell4 = AsposeUtil.getCell(cells, "身份证号码");
|
|
if (bsCell4 != null) {
|
|
if (excelFileName.endsWith(".xlsx")) {
|
|
personBs(bsCell4, excelFileName, sheetNo,nameWithSheetName);
|
|
}
|
|
|
|
if (excelFileName.endsWith(".xls")) {
|
|
readABCCustomerBS(excelFileName, cells, null);
|
|
}
|
|
|
|
templateIsExist = true;
|
|
}
|
|
}
|
|
|
|
if (!templateIsExist) {
|
|
Cell flagCell = AsposeUtil.getCell(cells, "中国农业银行司法查询控制系统");
|
|
if (flagCell != null) {
|
|
Cell temp = AsposeUtil.getCell(cells, "交易日期");
|
|
if (temp != null) {
|
|
personBs(temp, excelFileName, sheetNo,nameWithSheetName);
|
|
templateIsExist = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!templateIsExist && (!IGNORE_SHEET_NAME.contains(sheetName))) {
|
|
throw new TemplateNotFindException(sourceFile + BankStatementConstants.NAME_WITH_SHEET_NAME + sheetName);
|
|
}
|
|
} catch (Exception e) {
|
|
importResultService.record(caseId, BANK_NAME, e);
|
|
}
|
|
}
|
|
} catch (Exception e) {
|
|
importResultService.record(caseId, BANK_NAME, e);
|
|
}
|
|
}
|
|
}
|
|
|
|
private void readBsAndOai(String excelFileName, Cells cells,int sheetNo,String nameWithSheetName) {
|
|
Cell cardCell = AsposeUtil.getCell(cells, "产品号/账号/客户号");
|
|
if (cardCell == null) {
|
|
cardCell = AsposeUtil.getCell(cells, "卡号");
|
|
if(cardCell == null){
|
|
throw new TemplateNotFindException(nameWithSheetName);
|
|
}
|
|
}
|
|
String cardNumber = getCardNumber(cardCell,cells,nameWithSheetName);
|
|
String cardHolderName = getCardHolderName(cells,nameWithSheetName);
|
|
|
|
List<ABCCompanyInfoEntity> oaiList = new ArrayList<>();
|
|
try (ExcelReader reader = EasyExcel.read(excelFileName).build()) {
|
|
ReadSheet sheet = EasyExcel.readSheet(sheetNo)
|
|
.headRowNumber(cardCell.getRow()+1)
|
|
.head(ABCCompanyInfoEntity.class)
|
|
.registerReadListener(companyOAIReadListener(oaiList))
|
|
.build();
|
|
reader.read(sheet);
|
|
} catch (Exception e) {
|
|
throw new ImportDataFailedException(e.getMessage(), nameWithSheetName);
|
|
}
|
|
|
|
|
|
Cell tradeDateCell = AsposeUtil.getCell(cells, "交易日期");
|
|
if (tradeDateCell == null) {
|
|
tradeDateCell = AsposeUtil.getCell(cells, "发生额");//20240428
|
|
if(tradeDateCell == null){
|
|
throw new TemplateNotFindException(nameWithSheetName);
|
|
}
|
|
}
|
|
try (ExcelReader reader = EasyExcel.read(excelFileName).build()) {
|
|
ReadSheet sheet = EasyExcel.readSheet(sheetNo)
|
|
.headRowNumber(tradeDateCell.getRow() + 1)
|
|
.head(ABCCompanyStatementEntity.class)
|
|
.registerReadListener(companyBsListenerFor20240416(cardNumber,cardHolderName,nameWithSheetName))
|
|
.build();
|
|
reader.read(sheet);
|
|
} catch (Exception e) {
|
|
throw new ImportDataFailedException(e.getMessage(), nameWithSheetName);
|
|
}
|
|
|
|
}
|
|
|
|
|
|
private String getCardNumber(Cell cardCell,Cells cells,String sourceFile) {
|
|
|
|
int startRow = cardCell.getRow();
|
|
int startCol = cardCell.getColumn();
|
|
Cell cardNumberCell = cells.get(startRow + 1, startCol);
|
|
if(null == cardNumberCell || StrUtil.isEmpty(cardNumberCell.getStringValue())){
|
|
throw new ImportDataFailedException("读取卡号信息异常", sourceFile);
|
|
}
|
|
String cardNumber = cardNumberCell.getStringValue().trim();
|
|
if(cardNumber.contains("/")){
|
|
cardNumber = cardNumber.split("/")[0];
|
|
}
|
|
return cardNumber;
|
|
}
|
|
|
|
private String getCardHolderName(Cells cells,String sourceFile) {
|
|
Cell cardCell = AsposeUtil.getCell(cells, "户名");
|
|
if (cardCell == null) {
|
|
throw new ImportDataFailedException("无法读取表头信息.", sourceFile);
|
|
}
|
|
int startRow = cardCell.getRow();
|
|
int startCol = cardCell.getColumn();
|
|
Cell cardNumberCell = cells.get(startRow + 1, startCol);
|
|
if(null == cardNumberCell || StrUtil.isEmpty(cardNumberCell.getStringValue())){
|
|
throw new ImportDataFailedException("读取户名信息异常", sourceFile);
|
|
}
|
|
String cardNumber = cardNumberCell.getStringValue().trim();
|
|
if(cardNumber.contains("/")){
|
|
cardNumber = cardNumber.split("/")[0];
|
|
}
|
|
return cardNumber;
|
|
}
|
|
|
|
|
|
void personBs(Cell bsCell, String excelFileName, int sheetNo,String nameWithSheetName){
|
|
int headRowNum = bsCell.getRow() + 1;
|
|
|
|
try (ExcelReader reader = EasyExcel.read(excelFileName).build()) {
|
|
ReadSheet sheet = EasyExcel.readSheet(sheetNo)
|
|
.headRowNumber(headRowNum)
|
|
.head(ABCCustomerStatementEntity.class)
|
|
.registerReadListener(personBSReadListener(nameWithSheetName))
|
|
.build();
|
|
reader.read(sheet);
|
|
} catch (Exception e) {
|
|
throw new ImportDataFailedException(e.getMessage(), excelFileName);
|
|
}
|
|
}
|
|
|
|
private void companyBs(Cell tempCell, String excelFileName, int sheetNo,ABCCompanyInfoEntity oai) {
|
|
String sourceFile = HelperUtil.getSourceFileName(excelFileName,BANK_NAME);
|
|
int headRowNum = tempCell.getRow() + 1;
|
|
try (ExcelReader reader = EasyExcel.read(excelFileName).build()) {
|
|
ReadSheet sheet = EasyExcel.readSheet(sheetNo)
|
|
.headRowNumber(headRowNum)
|
|
.head(ABCCompanyStatementEntity.class)
|
|
.registerReadListener(companyBSReadListener(oai,sourceFile))
|
|
.build();
|
|
reader.read(sheet);
|
|
} catch (Exception e) {
|
|
throw new ImportDataFailedException(e.getMessage(), excelFileName);
|
|
}
|
|
}
|
|
|
|
|
|
private void companyOai(Cell tempCell, String excelFileName, int sheetNo,List<ABCCompanyInfoEntity> oaiList) {
|
|
int headRowNum = tempCell.getRow() + 1;
|
|
try (ExcelReader reader = EasyExcel.read(excelFileName).build()) {
|
|
ReadSheet sheet = EasyExcel.readSheet(sheetNo)
|
|
.headRowNumber(headRowNum)
|
|
.head(ABCCompanyInfoEntity.class)
|
|
.registerReadListener(companyOAIReadListener(oaiList))
|
|
.build();
|
|
reader.read(sheet);
|
|
|
|
|
|
} catch (Exception e) {
|
|
throw new ImportDataFailedException(e.getMessage(), excelFileName);
|
|
}
|
|
}
|
|
|
|
private void personOai(Cell oaiCell1, String excelFileName,int sheetNo,Cells cells) {
|
|
int headRowNum = oaiCell1.getRow() + 1;
|
|
try (ExcelReader reader = EasyExcel.read(excelFileName).build()) {
|
|
ReadSheet sheet = EasyExcel.readSheet(sheetNo)
|
|
.headRowNumber(headRowNum)
|
|
.head(ABCCustomerInfoEntity.class)
|
|
.registerReadListener(personOAIReadListener())
|
|
.build();
|
|
reader.read(sheet);
|
|
} catch (Exception e) {
|
|
if(e instanceof OLE2NotOfficeXmlFileException){
|
|
String sourceFile = HelperUtil.getSourceFileName(excelFileName,BANK_NAME);
|
|
log.error("xlsx是手动改的后缀 而不是通过另存为的方式-"+sourceFile);
|
|
try {
|
|
CellsWrapper<ABCCustomerInfoEntity> wrapper = new CellsWrapper<>(cells, ABCCustomerInfoEntity.class, false);
|
|
wrapper.doRead(-1);
|
|
List<ABCCustomerInfoEntity> entryList = wrapper.getEntryList();
|
|
esABCCustomerInfoMapper.insertBatch(entryList);
|
|
} catch (Exception ex) {
|
|
throw new ImportDataFailedException(ex.getMessage(), sourceFile);
|
|
}
|
|
|
|
}else {
|
|
throw new AnalyzeDataFailedException(e.getMessage(),e);
|
|
// HelperUtil.recordErrorImportResult(e.getMessage(), sourceFile,sheetName,caseId,BANK_NAME);
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
|
|
public ReadListener<ABCCompanyStatementEntity> companyBsListenerFor20240416(String cardNumber,String cardHolderName,String sourceFile) {
|
|
return new ReadListener<ABCCompanyStatementEntity>() {
|
|
|
|
List<ABCCompanyStatementEntity> cacheList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
|
|
@Override
|
|
public void invoke(ABCCompanyStatementEntity entity, AnalysisContext context) {
|
|
// 4 交易日期
|
|
String transDate = entity.getTransactionDate();
|
|
try {
|
|
if (StrUtil.isNotEmpty(transDate)) { // 20211006
|
|
String transTime = entity.getTransactionTime();
|
|
if (StrUtil.isNotEmpty(transTime)) { // 14:15:07
|
|
if (transTime.length() > Constants.DATE_TIME_FULL_FORMAT_LENGTH) {
|
|
// 20131001145718000
|
|
transTime = transTime.substring(8, 14);
|
|
DateUtil.parse(transTime, "HHmmss");
|
|
entity.setTransactionTime(transTime);
|
|
} else { // 20081222 030411
|
|
if (transTime.contains(":")) {
|
|
DateUtil.parse(transDate + " " + transTime, "yyyyMMdd HH:mm:ss");
|
|
} else {
|
|
DateUtil.parse(transDate + transTime, "yyyyMMddHHmmss");
|
|
}
|
|
}
|
|
} else {
|
|
DateUtil.parse(transDate, "yyyyMMdd");
|
|
}
|
|
}
|
|
} catch (Exception e) {
|
|
log.error(sourceFile+"解析交易日期异常", e);
|
|
Integer currentRowNum = context.getCurrentRowNum() + 1;
|
|
throw new AnalyzeDataFailedException("第"+ currentRowNum +"行解析交易日期异常", e, sourceFile);
|
|
}
|
|
entity.setId(IdUtil.objectId());
|
|
entity.setCardNumber(cardNumber);
|
|
entity.setCardHolderName(cardHolderName);
|
|
entity.setSourceFile(sourceFile);
|
|
cacheList.add(entity);
|
|
if (cacheList.size() >= Constants.BATCH_SIZE) {
|
|
save();
|
|
}
|
|
}
|
|
|
|
@Override
|
|
public void doAfterAllAnalysed(AnalysisContext context) {
|
|
if (!cacheList.isEmpty()) {
|
|
save();
|
|
}
|
|
}
|
|
|
|
private void save() {
|
|
esABCCompanyStatementMapper.insertBatch(cacheList);
|
|
cacheList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
}
|
|
};
|
|
}
|
|
|
|
|
|
public ReadListener<ABCCustomerInfoEntity> personOAIReadListener() {
|
|
return new ReadListener<ABCCustomerInfoEntity>() {
|
|
|
|
List<ABCCustomerInfoEntity> cacheList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
|
|
@Override
|
|
public void invoke(ABCCustomerInfoEntity entity, AnalysisContext context) {
|
|
entity.setId(IdUtil.objectId());
|
|
cacheList.add(entity);
|
|
if (cacheList.size() >= Constants.BATCH_SIZE) {
|
|
save();
|
|
}
|
|
}
|
|
|
|
@Override
|
|
public void doAfterAllAnalysed(AnalysisContext context) {
|
|
if (!cacheList.isEmpty()) {
|
|
save();
|
|
}
|
|
}
|
|
|
|
private void save() {
|
|
esABCCustomerInfoMapper.insertBatch(cacheList);
|
|
cacheList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
}
|
|
};
|
|
}
|
|
|
|
public ReadListener<ABCCompanyInfoEntity> companyOAIReadListener(List<ABCCompanyInfoEntity> oaiList) {
|
|
return new ReadListener<ABCCompanyInfoEntity>() {
|
|
|
|
List<ABCCompanyInfoEntity> cacheList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
|
|
@Override
|
|
public void invoke(ABCCompanyInfoEntity entity, AnalysisContext context) {
|
|
String cardNumber = entity.getCardNumber();
|
|
if(StrUtil.isEmpty(cardNumber)){
|
|
return;
|
|
}
|
|
if (HelperUtil.isContainsChinese(cardNumber)){
|
|
return;
|
|
}
|
|
cardNumber = entity.getCardNumber().trim();
|
|
if(cardNumber.contains("/")){
|
|
cardNumber = cardNumber.split("/")[0];
|
|
}
|
|
|
|
if(StrUtil.isEmpty(entity.getIdCardNo())){
|
|
entity.setIdCardNo("");//做统计值是null不会被统计,空字符串可以
|
|
}
|
|
|
|
entity.setCardNumber(cardNumber);
|
|
entity.setId(IdUtil.objectId());
|
|
cacheList.add(entity);
|
|
oaiList.add(entity);
|
|
if (cacheList.size() >= Constants.BATCH_SIZE) {
|
|
save();
|
|
}
|
|
}
|
|
|
|
@Override
|
|
public void doAfterAllAnalysed(AnalysisContext context) {
|
|
if (!cacheList.isEmpty()) {
|
|
save();
|
|
}
|
|
}
|
|
|
|
private void save() {
|
|
esABCCompanyInfoMapper.insertBatch(cacheList);
|
|
cacheList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
}
|
|
};
|
|
}
|
|
|
|
public ReadListener<ABCCompanyStatementEntity> companyBSReadListener(ABCCompanyInfoEntity oai,String sourceFile) {
|
|
return new ReadListener<ABCCompanyStatementEntity>() {
|
|
|
|
List<ABCCompanyStatementEntity> cacheList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
|
|
@Override
|
|
public void invoke(ABCCompanyStatementEntity entity, AnalysisContext context) {
|
|
entity.setId(IdUtil.objectId());
|
|
entity.setSourceFile(sourceFile);
|
|
if(null != oai && StrUtil.isEmpty(entity.getCardNumber())){
|
|
entity.setCardNumber(oai.getCardNumber2());
|
|
entity.setCardHolderName(oai.getCardHolderName());
|
|
}
|
|
cacheList.add(entity);
|
|
if (cacheList.size() >= Constants.BATCH_SIZE) {
|
|
save();
|
|
}
|
|
}
|
|
|
|
@Override
|
|
public void doAfterAllAnalysed(AnalysisContext context) {
|
|
if (!cacheList.isEmpty()) {
|
|
save();
|
|
}
|
|
}
|
|
|
|
private void save() {
|
|
esABCCompanyStatementMapper.insertBatch(cacheList);
|
|
cacheList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
}
|
|
};
|
|
}
|
|
|
|
public ReadListener<ABCCustomerStatementEntity> personBSReadListener(String sourceFile) {
|
|
return new ReadListener<ABCCustomerStatementEntity>() {
|
|
|
|
List<ABCCustomerStatementEntity> cacheList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
|
|
@Override
|
|
public void invoke(ABCCustomerStatementEntity entity, AnalysisContext context) {
|
|
entity.setId(IdUtil.objectId());
|
|
entity.setSourceFile(sourceFile);
|
|
cacheList.add(entity);
|
|
if (cacheList.size() >= Constants.BATCH_SIZE) {
|
|
save();
|
|
}
|
|
}
|
|
|
|
@Override
|
|
public void doAfterAllAnalysed(AnalysisContext context) {
|
|
if (!cacheList.isEmpty()) {
|
|
save();
|
|
}
|
|
}
|
|
|
|
private void save() {
|
|
esABCCustomerStatementMapper.insertBatch(cacheList);
|
|
cacheList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
}
|
|
};
|
|
}
|
|
|
|
public void analyzeData(String caseId) {
|
|
|
|
// 开户信息
|
|
analyzeOAI(caseId);
|
|
// 流水信息
|
|
analyzeBS(caseId);
|
|
}
|
|
|
|
private void analyzeOAI(String caseId) {
|
|
// 先分析公司数据
|
|
analyzeCompanyOAI(caseId);
|
|
analyzeCustomerOAI(caseId);
|
|
}
|
|
|
|
private void analyzeCustomerOAI(String caseId) {
|
|
List<OpeningAccountInfo> oaiList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
|
|
List<ABCCustomerInfoEntity> entityList =
|
|
HelperUtil.getEntityList(esABCCustomerInfoMapper, ABCCustomerInfoEntity.class);
|
|
Set<String> uniqueKeySet = new HashSet();
|
|
for (ABCCustomerInfoEntity entity : entityList) {
|
|
|
|
OpeningAccountInfo oai = getOpeningAccountInfo( entity);
|
|
String md5Id = HelperUtil.generateMD5Id4OAI(oai,caseId);
|
|
//未导入数据内部去重
|
|
if(HelperUtil.deduplication(md5Id,uniqueKeySet)){
|
|
continue;
|
|
}
|
|
|
|
oai.setId(md5Id);
|
|
oai.setCaseId(caseId);
|
|
oaiList.add(oai);
|
|
if (oaiList.size() >= Constants.BATCH_SIZE) {
|
|
List<OpeningAccountInfo> dest = HelperUtil.getDest(oaiList);
|
|
HelperUtil.batchSaveOAI2Es(dest, caseId);
|
|
|
|
oaiList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
}
|
|
}
|
|
uniqueKeySet.clear();
|
|
if (!oaiList.isEmpty()) {
|
|
List<OpeningAccountInfo> dest = HelperUtil.getDest(oaiList);
|
|
HelperUtil.batchSaveOAI2Es(dest, caseId);
|
|
}
|
|
}
|
|
|
|
private static OpeningAccountInfo getOpeningAccountInfo(ABCCustomerInfoEntity entity) {
|
|
OpeningAccountInfo oai = new OpeningAccountInfo();
|
|
oai.setBankName(BANK_NAME);
|
|
|
|
String cardNumber = entity.getCardNumber();
|
|
oai.setAccountNumber(cardNumber);
|
|
oai.setIdNo(entity.getIdCardNo());
|
|
|
|
oai.setName(entity.getCardHolderName());
|
|
|
|
oai.setPhone(entity.getPhone());
|
|
oai.setAddress(entity.getAddress());
|
|
oai.setOpeningAccountDate(entity.getOpeningAccountDate());
|
|
oai.setClosingDate(entity.getCloseDate());
|
|
oai.setStatus(entity.getStatus());
|
|
oai.setAccountOpeningInstitution(entity.getAccountOpeningInstitution());
|
|
oai.setBalance(entity.getBalance());
|
|
|
|
return oai;
|
|
}
|
|
|
|
private void analyzeCompanyOAI(String caseId) {
|
|
|
|
List<OpeningAccountInfo> oaiList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
|
|
List<ABCCompanyInfoEntity> entityList =
|
|
HelperUtil.getEntityList(esABCCompanyInfoMapper, ABCCompanyInfoEntity.class);
|
|
Set<String> uniqueKeySet = new HashSet();
|
|
for (ABCCompanyInfoEntity entity : entityList) {
|
|
|
|
OpeningAccountInfo oai = getOpeningAccountInfo(entity);
|
|
|
|
String md5Id = HelperUtil.generateMD5Id4OAI(oai,caseId);
|
|
//未导入数据内部去重
|
|
if(HelperUtil.deduplication(md5Id,uniqueKeySet)){
|
|
continue;
|
|
}
|
|
|
|
oai.setId(md5Id);
|
|
oai.setCaseId(caseId);
|
|
oaiList.add(oai);
|
|
if (oaiList.size() >= Constants.BATCH_SIZE) {
|
|
List<OpeningAccountInfo> dest = HelperUtil.getDest(oaiList);
|
|
HelperUtil.batchSaveOAI2Es(dest, caseId);
|
|
|
|
oaiList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
}
|
|
}
|
|
|
|
if (!oaiList.isEmpty()) {
|
|
List<OpeningAccountInfo> dest = HelperUtil.getDest(oaiList);
|
|
HelperUtil.batchSaveOAI2Es(dest, caseId);
|
|
}
|
|
}
|
|
|
|
private static OpeningAccountInfo getOpeningAccountInfo(ABCCompanyInfoEntity entity) {
|
|
OpeningAccountInfo oai = new OpeningAccountInfo();
|
|
oai.setName(entity.getCardHolderName());
|
|
oai.setBankName(BANK_NAME);
|
|
oai.setAccountNumber(entity.getCardNumber());
|
|
oai.setPhone(entity.getPhone());
|
|
oai.setAddress(entity.getAddress());
|
|
oai.setOpeningAccountDate(entity.getOpeningAccountDate());
|
|
oai.setClosingDate(entity.getCloseDate());
|
|
oai.setStatus(entity.getStatus());
|
|
oai.setAccountOpeningInstitution(entity.getAccountOpeningInstitution());
|
|
return oai;
|
|
}
|
|
|
|
private void analyzeBS(String caseId) {
|
|
// 异步进行
|
|
analyzeBSForCompany(caseId);
|
|
analyzeBSForCustomer(caseId);
|
|
}
|
|
|
|
private void analyzeBSForCustomer(String caseId) {
|
|
//去重
|
|
Set<String> uniqueKeySet = new HashSet();
|
|
List<PlateNumberInfo> plateNumberInfoList = new ArrayList<>();
|
|
|
|
List<BankStatement> bsList = new ArrayList<>();
|
|
|
|
|
|
Map<String, ABCCustomerInfoEntity> cache = new HashMap<>();
|
|
|
|
List<OpeningAccountInfo> oaiList = HelperUtil.getEntityListV2(esOpeningAccountInfoMapper, OpeningAccountInfo.class, caseId);
|
|
|
|
List<ABCCustomerInfoEntity> customerInfoList =
|
|
HelperUtil.getEntityList(esABCCustomerInfoMapper, ABCCustomerInfoEntity.class);
|
|
for (ABCCustomerInfoEntity info : customerInfoList) {
|
|
cache.put(info.getIdCardNo(), info);
|
|
}
|
|
|
|
Map<String, List<ABCCustomerInfoEntity>> groupByCardNumber = customerInfoList.stream().filter(item ->StrUtil.isNotEmpty(item.getCardNumber()))
|
|
.collect(Collectors.groupingBy(ABCCustomerInfoEntity::getCardNumber));
|
|
Map<String, List<ABCCustomerInfoEntity>> groupByAccountNo = customerInfoList.stream().filter(item ->StrUtil.isNotEmpty(item.getAccountNo()))
|
|
.collect(Collectors.groupingBy(ABCCustomerInfoEntity::getAccountNo));
|
|
// 处理个人流水
|
|
List<ABCCustomerStatementEntity> entityList =
|
|
HelperUtil.getEntityList(esABCCustomerStatementMapper, ABCCustomerStatementEntity.class);
|
|
entityList.stream().filter(item ->StrUtil.isNotEmpty(item.getAccountNo()))
|
|
.collect(Collectors.groupingBy(ABCCustomerStatementEntity::getAccountNo));
|
|
// 主要其实就是流水数据
|
|
for (ABCCustomerStatementEntity entity : entityList) {
|
|
String sourceFile = entity.getSourceFile();
|
|
try {
|
|
// 流水
|
|
BankStatement bs = new BankStatement();
|
|
bs.setBankName(BANK_NAME);
|
|
String cardNumber = entity.getCardNumber();
|
|
String accountNo = entity.getAccountNo();
|
|
bs.setIdCardNo(entity.getIdCardNo());
|
|
bs.setCardHolderName(entity.getCardHolderName());
|
|
bs.setCardNumber(cardNumber);
|
|
if (StrUtil.isEmpty(bs.getIdCardNo()) || StrUtil.isEmpty(bs.getCardHolderName())) {
|
|
if (groupByCardNumber.containsKey(cardNumber)) {
|
|
List<ABCCustomerInfoEntity> abcCustomerInfoEntities = groupByCardNumber.get(cardNumber);
|
|
ABCCustomerInfoEntity oai = abcCustomerInfoEntities.get(0);
|
|
bs.setIdCardNo(oai.getIdCardNo());
|
|
bs.setCardHolderName(oai.getCardHolderName());
|
|
bs.setCardNumber(StringUtils.isEmpty(oai.getCardNumber())?oai.getCardNumber2():oai.getCardNumber());
|
|
}
|
|
|
|
if(StrUtil.isEmpty(bs.getCardHolderName()) && StrUtil.isEmpty(entity.getCardHolderName()) && StrUtil.isNotEmpty(accountNo) ){
|
|
for (String oaiAccountNo : groupByAccountNo.keySet()) {
|
|
if(oaiAccountNo.contains(accountNo)){
|
|
List<ABCCustomerInfoEntity> abcCustomerInfoEntities = groupByAccountNo.get(oaiAccountNo);
|
|
ABCCustomerInfoEntity oai = abcCustomerInfoEntities.get(0);
|
|
bs.setIdCardNo(oai.getIdCardNo());
|
|
bs.setCardHolderName(oai.getCardHolderName());
|
|
bs.setCardNumber(StringUtils.isEmpty(oai.getCardNumber())?oai.getCardNumber2():oai.getCardNumber());
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
ABCCustomerInfoEntity info = cache.getOrDefault(bs.getIdCardNo(), null);
|
|
if (info != null) {
|
|
bs.setPhone(info.getPhone());
|
|
if (StringUtils.isEmpty(bs.getCardHolderName())) {
|
|
bs.setCardHolderName(info.getCardHolderName());
|
|
if (StringUtils.isEmpty(bs.getCardHolderName())) {
|
|
Optional<ABCCustomerInfoEntity> first = cache.values().stream().filter(item -> StrUtil.isNotEmpty(item.getCardNumber())).filter(cs -> cs.getCardNumber().equalsIgnoreCase(bs.getCardNumber())).findFirst();
|
|
first.ifPresent(abcCustomerInfoEntity -> bs.setCardHolderName(abcCustomerInfoEntity.getCardHolderName()));
|
|
first.ifPresent(abcCustomerInfoEntity -> bs.setIdCardNo(abcCustomerInfoEntity.getIdCardNo()));
|
|
}
|
|
}
|
|
|
|
if (StringUtils.isEmpty(bs.getCardNumber())) {
|
|
bs.setCardNumber(info.getCardNumber());
|
|
if (StringUtils.isEmpty(bs.getCardNumber())) {
|
|
Optional<ABCCustomerInfoEntity> first = cache.values().stream().filter(item -> StrUtil.isNotEmpty(item.getCardNumber())).filter(cs -> cs.getIdCardNo().equalsIgnoreCase(bs.getIdCardNo())).findFirst();
|
|
first.ifPresent(abcCustomerInfoEntity -> bs.setCardNumber(abcCustomerInfoEntity.getCardNumber()));
|
|
}
|
|
}
|
|
|
|
|
|
cache.put(info.getIdCardNo(), info);
|
|
}
|
|
|
|
if (StringUtils.isEmpty(bs.getIdCardNo()) && StringUtils.isNotEmpty(bs.getCardNumber())) {
|
|
Optional<OpeningAccountInfo> first = oaiList.stream().filter(cs -> StrUtil.isNotEmpty(cs.getAccountNumber())).filter(cs -> cs.getAccountNumber().equalsIgnoreCase(bs.getCardNumber())).findFirst();
|
|
first.ifPresent(abcCustomerInfoEntity -> bs.setIdCardNo(abcCustomerInfoEntity.getIdNo()));
|
|
}
|
|
// 1 货币类型
|
|
// 人民币
|
|
bs.setTransCurrencyType(Constants.CURRENCY_TYPE_CHINA);
|
|
|
|
// 2 交易金额 可能是double或者int类型
|
|
bs.setTransactionAmount(entity.getTransactionAmount());
|
|
|
|
// 3 余额
|
|
bs.setBalance(entity.getBalance());
|
|
// 4 交易时间
|
|
String transactionDate = entity.getTransactionDate();
|
|
String transTime = entity.getTransactionTime();
|
|
if (StrUtil.isNotEmpty(transactionDate)) {
|
|
|
|
if (StringUtils.isEmpty(transTime)) {
|
|
transTime = "000000";
|
|
}
|
|
if(!isInteger(transTime)){
|
|
transTime = "000000";
|
|
}
|
|
|
|
String transactionTime = transactionDate + transTime;
|
|
String format = "yyyyMMddHHmmss";
|
|
try {
|
|
|
|
if (transactionTime.length() > format.length()) {
|
|
transactionTime = transactionTime.substring(0, format.length());
|
|
}
|
|
|
|
bs.setTransactionTime(DateUtil.parse(transactionTime, format));
|
|
} catch (Exception e) {
|
|
|
|
String filename = entity.getSourceFile();
|
|
|
|
log.error("解析交易时间异常: {}{}", e.getMessage(), e,filename);
|
|
throw new AnalyzeDataFailedException(String.format("解析交易时间错误，无法将【%s】格式化为【%s】", transactionTime, format), e, sourceFile);
|
|
}
|
|
}
|
|
|
|
String counterpartyName = entity.getCounterpartyName();
|
|
String counterpartyIdCardNo = entity.getCounterpartyIdCardNo();
|
|
bs.setCounterpartIdCardNo(counterpartyIdCardNo);
|
|
if (StrUtil.isNotEmpty(counterpartyName) && StrUtil.isEmpty(counterpartyIdCardNo)) {
|
|
List<ABCCustomerInfoEntity> rstList = cache.values().stream()
|
|
.filter(cs -> cs.getCardHolderName().equalsIgnoreCase(counterpartyName))
|
|
.collect(Collectors.toList());
|
|
if (!rstList.isEmpty()) {
|
|
ABCCustomerInfoEntity cs = rstList.get(0);
|
|
bs.setCounterpartIdCardNo(cs.getIdCardNo());
|
|
}
|
|
}
|
|
|
|
bs.setCounterpartyName(counterpartyName);
|
|
bs.setCounterpartyBankName(entity.getCounterpartyBankName());
|
|
bs.setCounterpartyAccount(entity.getCounterpartyAccount());
|
|
bs.setTransactionInstitutions(entity.getTransactionInstitutions());
|
|
bs.setSummary(entity.getSummary());
|
|
bs.setTransRemark(entity.getTransRemark());
|
|
bs.setTransChannel(entity.getTransChannel());
|
|
|
|
String md5Id = HelperUtil.generateMD5Id(bs,caseId);
|
|
//未导入数据内部去重
|
|
if(HelperUtil.deduplication(md5Id,uniqueKeySet)){
|
|
continue;
|
|
}
|
|
|
|
bs.setSourceFile(entity.getSourceFile());
|
|
bs.setId(md5Id);
|
|
bs.setCaseId(caseId);
|
|
try {
|
|
BeanUtils.beanAttributeValueTrim(bs);
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
}
|
|
bsList.add(bs);
|
|
HelperUtil.extractPlateNumber(bs,plateNumberInfoList);
|
|
|
|
|
|
if (bsList.size() >= Constants.BATCH_SIZE) {
|
|
List<BankStatement> dest = HelperUtil.getDest(bsList);
|
|
HelperUtil.batchInsert2Es(dest, caseId);
|
|
|
|
bsList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
}
|
|
} catch (Exception e) {
|
|
importResultService.record(caseId, BANK_NAME, e, sourceFile);
|
|
}
|
|
}
|
|
|
|
uniqueKeySet.clear();
|
|
HelperUtil.batchInsertPlateNumber(plateNumberInfoList);
|
|
|
|
// 保存数据库
|
|
if (!bsList.isEmpty()) {
|
|
List<BankStatement> dest = HelperUtil.getDest(bsList);
|
|
HelperUtil.batchInsert2Es(dest, caseId);
|
|
}
|
|
}
|
|
|
|
private void analyzeBSForCompany(String caseId) {
|
|
List<BankStatement> bsList = new ArrayList<>();
|
|
|
|
|
|
List<ABCCompanyStatementEntity> entityList =
|
|
HelperUtil.getEntityList(esABCCompanyStatementMapper, ABCCompanyStatementEntity.class);
|
|
|
|
|
|
List<ABCCompanyInfoEntity> companyInfoList =
|
|
HelperUtil.getEntityList(esABCCompanyInfoMapper, ABCCompanyInfoEntity.class);
|
|
Map<String, ABCCompanyInfoEntity> nameCompanyInfoMap = new HashMap<>();
|
|
|
|
for (ABCCompanyInfoEntity entity : companyInfoList) {
|
|
nameCompanyInfoMap.put(entity.getCardHolderName(), entity);
|
|
}
|
|
|
|
Map<String, List<ABCCompanyInfoEntity>> groupByContractNo = companyInfoList.stream()
|
|
.filter(item ->StrUtil.isNotEmpty(item.getContractNo()))
|
|
.collect(Collectors.groupingBy(ABCCompanyInfoEntity::getContractNo));
|
|
|
|
|
|
Set<String> uniqueKeySet = new HashSet();
|
|
List<PlateNumberInfo> plateNumberInfoList = new ArrayList<>();
|
|
|
|
for (ABCCompanyStatementEntity entity : entityList) {
|
|
String sourceFile = entity.getSourceFile();
|
|
try {
|
|
// 流水
|
|
BankStatement bs = new BankStatement();
|
|
bs.setBankName(BANK_NAME);
|
|
|
|
String cardHolderName = entity.getCardHolderName();
|
|
bs.setCardHolderName(cardHolderName);
|
|
bs.setCardNumber(entity.getCardNumber());
|
|
if (StrUtil.isEmpty(bs.getCardHolderName())) {
|
|
if (groupByContractNo.containsKey(entity.getContractNo())) {
|
|
List<ABCCompanyInfoEntity> abcCompanyInfoEntities = groupByContractNo.get(entity.getContractNo());
|
|
ABCCompanyInfoEntity companyInfo = abcCompanyInfoEntities.get(0);
|
|
bs.setCardHolderName(companyInfo.getCardHolderName());
|
|
String cardNumber = companyInfo.getCardNumber();
|
|
cardNumber = StrUtil.isEmpty(cardNumber) ? companyInfo.getCardNumber2() : cardNumber;
|
|
bs.setCardNumber(StrUtil.isEmpty(cardNumber) ? companyInfo.getContractNo() : cardNumber);
|
|
bs.setIdCardNo(companyInfo.getIdCardNo());
|
|
}
|
|
}
|
|
if (StrUtil.isEmpty(bs.getIdCardNo())) {
|
|
bs.setIdCardNo("");
|
|
}
|
|
ABCCompanyInfoEntity ci = nameCompanyInfoMap.getOrDefault(cardHolderName, null);
|
|
if (ci != null) {
|
|
if (StrUtil.isEmpty(bs.getCardNumber())) {
|
|
bs.setCardNumber(ci.getCardNumber());
|
|
}
|
|
bs.setPhone(ci.getPhone());
|
|
}
|
|
// 1 货币类型
|
|
// 人民币
|
|
bs.setTransCurrencyType(Constants.CURRENCY_TYPE_CHINA);
|
|
|
|
// 2 交易金额 可能是double或者int类型
|
|
bs.setTransactionAmount(entity.getTransactionAmount());
|
|
|
|
// 3 余额
|
|
bs.setBalance(entity.getBalance());
|
|
// 4 交易日期 20170314143047165086
|
|
String transactionDate = entity.getTransactionDate();
|
|
if (StrUtil.isNotEmpty(transactionDate)) {
|
|
// 如果包含-，则用yyyy-MM-dd来解析
|
|
String format;
|
|
if (transactionDate.contains("-")) {
|
|
format = "yyyy-MM-dd";
|
|
} else if (transactionDate.contains("/")) {
|
|
format = "yyyy/MM/dd";
|
|
} else {
|
|
format = "yyyyMMdd";
|
|
}
|
|
|
|
String transactionTime = entity.getTransactionTime();
|
|
// 如果不为空，则组合
|
|
if (StrUtil.isNotEmpty(transactionTime)) {
|
|
if (!isInteger(transactionTime)) {
|
|
transactionTime = "00:00:00";
|
|
}
|
|
// 还需要分情况考虑，一种是交易时间的长度超过了14位，另一种是交易时间的长度小于14位
|
|
if (transactionTime.length() > Constants.DATE_TIME_FULL_FORMAT_LENGTH) {
|
|
transactionTime = transactionTime.substring(
|
|
Constants.DATE_FULL_FORMAT_LENGTH, Constants.DATE_TIME_FULL_FORMAT_LENGTH);
|
|
}
|
|
|
|
transactionDate += " " + transactionTime;
|
|
// 如果包含：，则用HH:mm:ss来解析
|
|
if (transactionTime.contains(":")) {
|
|
format = format + " HH:mm:ss";
|
|
} else {
|
|
format += " HHmmss";
|
|
}
|
|
|
|
try {
|
|
bs.setTransactionTime(DateUtil.parse(transactionDate, format));
|
|
} catch (Exception e) {
|
|
log.error("解析交易日期异常:{},{}", e.getMessage() + entity.getSourceFile(), e);
|
|
throw new AnalyzeDataFailedException(
|
|
"解析交易日期失败，无法将【" + transactionDate + transactionTime + "】以格式【" + format + "】解析", e, entity.getSourceFile());
|
|
}
|
|
}
|
|
} else {
|
|
// 一般情况下，交易日期不为空
|
|
// 若是为空，则交易日期和时间同时包含在交易时间内哦
|
|
String transactionTime = entity.getTransactionTime();
|
|
if (transactionTime != null) {
|
|
|
|
String format = "yyyyMMddHHmmss";
|
|
|
|
try {
|
|
// 需要截断 20170314 143047
|
|
int len = format.length();
|
|
if (transactionTime.length() > len) {
|
|
transactionTime = transactionTime.substring(0, len);
|
|
}
|
|
bs.setTransactionTime(DateUtil.parse(transactionTime, format));
|
|
} catch (Exception e) {
|
|
log.error("解析交易日期异常:{}", e.getMessage(), e);
|
|
|
|
|
|
throw new AnalyzeDataFailedException(
|
|
"解析交易日期失败，无法将【" + transactionTime + "】以格式【" + format + "】解析", e, sourceFile);
|
|
}
|
|
}
|
|
}
|
|
|
|
String counterpartyName = entity.getCounterpartyName();
|
|
bs.setCounterpartyName(counterpartyName);
|
|
|
|
|
|
bs.setCounterpartyAccount(entity.getCounterpartyAccount());
|
|
bs.setCounterpartyBankName(entity.getCounterpartyBankName());
|
|
bs.setTransChannel(entity.getTransChannel());
|
|
bs.setSummary(entity.getSummary());
|
|
bs.setTransactionInstitutions(entity.getTransactionInstitutions());
|
|
|
|
String md5Id = HelperUtil.generateMD5Id(bs, caseId);
|
|
//未导入数据内部去重
|
|
if (HelperUtil.deduplication(md5Id, uniqueKeySet)) {
|
|
continue;
|
|
}
|
|
|
|
bs.setSourceFile(entity.getSourceFile());
|
|
bs.setId(md5Id);
|
|
bs.setCaseId(caseId);
|
|
try {
|
|
BeanUtils.beanAttributeValueTrim(bs);
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
}
|
|
bsList.add(bs);
|
|
HelperUtil.extractPlateNumber(bs,plateNumberInfoList);
|
|
|
|
|
|
if (bsList.size() >= Constants.BATCH_SIZE) {
|
|
List<BankStatement> dest = HelperUtil.getDest(bsList);
|
|
HelperUtil.batchInsert2Es(dest, caseId);
|
|
|
|
bsList.clear();
|
|
bsList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
|
|
}
|
|
} catch (Exception e) {
|
|
importResultService.record(caseId, BANK_NAME, e, sourceFile);
|
|
}
|
|
}
|
|
|
|
uniqueKeySet.clear();
|
|
HelperUtil.batchInsertPlateNumber(plateNumberInfoList);
|
|
// 保存数据库
|
|
if (!bsList.isEmpty()) {
|
|
List<BankStatement> dest = HelperUtil.getDest(bsList);
|
|
HelperUtil.batchInsert2Es(dest, caseId);
|
|
}
|
|
}
|
|
|
|
public static boolean isInteger(String str) {
|
|
Pattern pattern = Pattern.compile("^[-\\+]?[\\d]*$");
|
|
return pattern.matcher(str).matches();
|
|
}
|
|
|
|
private List<ABCCustomerInfoEntity> readOAI( Cells cells) throws Exception {
|
|
CellsWrapper<ABCCustomerInfoEntity> wrapper = new CellsWrapper<>(cells, ABCCustomerInfoEntity.class, false);
|
|
wrapper.doRead(-1);
|
|
List<ABCCustomerInfoEntity> entryList = wrapper.getEntryList();
|
|
esABCCustomerInfoMapper.insertBatch(entryList);
|
|
return entryList;
|
|
}
|
|
|
|
private List<ABCCompanyInfoEntity> readCompanyOAIByXls(Cells cells) throws Exception {
|
|
CellsWrapper<ABCCompanyInfoEntity> wrapper = new CellsWrapper<>(cells, ABCCompanyInfoEntity.class, false);
|
|
wrapper.doRead(-1);
|
|
List<ABCCompanyInfoEntity> entryList = wrapper.getEntryList();
|
|
esABCCompanyInfoMapper.insertBatch(entryList);
|
|
return entryList;
|
|
}
|
|
|
|
private void readABCCustomerBS(String excelFileName, Cells cells, List<ABCCustomerInfoEntity> oaiList) throws Exception {
|
|
String sourceFile = HelperUtil.getSourceFileName(excelFileName,BANK_NAME);
|
|
|
|
// 读个人流水
|
|
try {
|
|
CellsWrapper<ABCCustomerStatementEntity> wrapper = new CellsWrapper<>(cells, ABCCustomerStatementEntity.class, true);
|
|
wrapper.doRead(-1);
|
|
List<ABCCustomerStatementEntity> entryList = wrapper.getEntryList();
|
|
Map<String, List<ABCCustomerInfoEntity>> groupByAccountNo = new HashMap<>();
|
|
if(CollectionUtil.isNotEmpty(oaiList)){
|
|
groupByAccountNo = oaiList.stream().collect(Collectors.groupingBy(ABCCustomerInfoEntity::getAccountNo));
|
|
}
|
|
for (ABCCustomerStatementEntity bs : entryList) {
|
|
bs.setId(IdUtil.objectId());
|
|
bs.setSourceFile(sourceFile);
|
|
String accountNo = bs.getAccountNo();
|
|
if(StrUtil.isNotEmpty(accountNo)){
|
|
if (groupByAccountNo.containsKey(accountNo)) {
|
|
List<ABCCustomerInfoEntity> tempList = groupByAccountNo.get(accountNo);
|
|
ABCCustomerInfoEntity oai = tempList.get(0);
|
|
bs.setCardHolderName(oai.getCardHolderName());
|
|
bs.setCardNumber(oai.getCardNumber());
|
|
}
|
|
}
|
|
}
|
|
|
|
//save to es
|
|
esABCCustomerStatementMapper.insertBatch(entryList);
|
|
|
|
} catch (Exception e) {
|
|
|
|
if (e instanceof ExcelAnalysisStopException) {
|
|
return;
|
|
}
|
|
|
|
log.error("readABCCustomerBS:读取文件异常:{}", e.getMessage(), e);
|
|
throw new ImportDataFailedException(e.getMessage(), excelFileName);
|
|
}
|
|
}
|
|
|
|
private void readCompanyBSByXls(String excelFileName, Cells cells,ABCCompanyInfoEntity oai) throws Exception {
|
|
String sourceFile = HelperUtil.getSourceFileName(excelFileName,BANK_NAME);
|
|
|
|
try {
|
|
|
|
CellsWrapper<ABCCompanyStatementEntity> wrapper = new CellsWrapper<>(cells, ABCCompanyStatementEntity.class, true);
|
|
wrapper.doRead(-1);
|
|
String cardHolderName = oai.getCardHolderName();
|
|
String cardNumber = oai.getCardNumber();
|
|
if(StrUtil.isEmpty(cardNumber)){
|
|
cardNumber = oai.getCardNumber2();
|
|
}else {
|
|
if(cardNumber.length() > 17){
|
|
cardNumber = oai.getCardNumber2();
|
|
}
|
|
}
|
|
List<ABCCompanyStatementEntity> entryList = wrapper.getEntryList();
|
|
//加ID
|
|
for (ABCCompanyStatementEntity bs : entryList) {
|
|
bs.setId(IdUtil.objectId());
|
|
bs.setSourceFile(sourceFile);
|
|
bs.setCardNumber(cardNumber);
|
|
bs.setCardHolderName(cardHolderName);
|
|
}
|
|
|
|
//save to es
|
|
esABCCompanyStatementMapper.insertBatch(entryList);
|
|
|
|
} catch (Exception e) {
|
|
log.error("readABCCompanyBS:读取文件异常:{}", e.getMessage(), e);
|
|
throw new ImportDataFailedException(e.getMessage(), excelFileName);
|
|
}
|
|
}
|
|
|
|
}
|