case-analysis-service/cas-system/src/main/java/com/inscloudtech/bankStatementAnalysis/helper/ABCDataAnalysisHelper.java

package com.inscloudtech.bankStatementAnalysis.helper;

import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.IdUtil;
import cn.hutool.core.util.StrUtil;
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelReader;
import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.exception.ExcelAnalysisStopException;
import com.alibaba.excel.read.listener.ReadListener;
import com.alibaba.excel.read.metadata.ReadSheet;
import com.alibaba.excel.util.ListUtils;
import com.aspose.cells.Cell;
import com.aspose.cells.Cells;
import com.aspose.cells.Workbook;
import com.aspose.cells.Worksheet;
import com.baomidou.mybatisplus.core.toolkit.StringUtils;

import com.inscloudtech.bankStatementAnalysis.excel.CellsWrapper;
import com.inscloudtech.bankStatementAnalysis.helper.HelperUtil;
import com.inscloudtech.bankStatementAnalysis.mapper.*;
import com.inscloudtech.bankStatementAnalysis.service.ImportService;
import com.inscloudtech.common.constant.BankStatementConstants;
import com.inscloudtech.common.constant.Constants;
import com.inscloudtech.common.exception.dc.AnalyzeDataFailedException;
import com.inscloudtech.common.exception.dc.ImportDataFailedException;
import com.inscloudtech.common.exception.dc.TemplateNotFindException;
import com.inscloudtech.common.utils.bean.BeanUtils;
import com.inscloudtech.common.utils.file.FileUtils;
import com.inscloudtech.datacenter.domain.PlateNumberInfo;
import com.inscloudtech.datacenter.mapper.es.*;
import com.inscloudtech.datacenter.service.ImportResultService;
import com.inscloudtech.datacenter.domain.BankStatement;
import com.inscloudtech.datacenter.domain.OpeningAccountInfo;

import com.inscloudtech.bankStatementAnalysis.domain.entity.ABCCompanyInfoEntity;
import com.inscloudtech.bankStatementAnalysis.domain.entity.ABCCompanyStatementEntity;
import com.inscloudtech.bankStatementAnalysis.domain.entity.ABCCustomerInfoEntity;
import com.inscloudtech.bankStatementAnalysis.domain.entity.ABCCustomerStatementEntity;

import com.inscloudtech.bankStatementAnalysis.util.AsposeUtil;

import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.poi.openxml4j.exceptions.OLE2NotOfficeXmlFileException;
import org.springframework.stereotype.Component;

import java.io.File;
import java.util.*;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

/**
 * 农业银行数据分析
 */
@Slf4j
@RequiredArgsConstructor
@Component
public class ABCDataAnalysisHelper {

    private final ImportResultService importResultService;
    private final ImportService importService;

    private final EsABCCustomerStatementMapper esABCCustomerStatementMapper;
    private final EsABCCompanyStatementMapper esABCCompanyStatementMapper;
    private final EsABCCustomerInfoMapper esABCCustomerInfoMapper;
    private final EsABCCompanyInfoMapper esABCCompanyInfoMapper;
    private final ESOpeningAccountInfoMapper esOpeningAccountInfoMapper;
    private final static String BANK_NAME = "农业银行";
    private final static String IGNORE_SHEET_NAME = "金融资产列表,首页,子账户列表,冻结历史列表,共有权列表";

    public void importData(File file, String caseId) {
        List<File> fileList = FileUtil.loopFiles(file);

        // 筛选文件
        List<File> excelFileList = HelperUtil.getExcelFile(fileList);

        //农行数据大致分为两类：
        // 1. 开户信息和流水在同一个工作簿
        // 2. 开户信息和流水分开工作簿
        //农行的数据文件分两类：1、无法读取；2、可以读取
        for (File excelFile : excelFileList) {
            try {
                String excelFileName = excelFile.getAbsolutePath();
                String sourceFile = HelperUtil.getSourceFileName(excelFileName, BANK_NAME);
                Workbook wb = new Workbook(excelFileName);
                List<ABCCompanyInfoEntity> companyOAIList = new ArrayList<>();
                boolean bsHasNotCard = false;

                int count = wb.getWorksheets().getCount();
                for (int sheetNo = 0; sheetNo < count; sheetNo++) {
                    try {
                        boolean templateIsExist = false;
                        Worksheet worksheet = wb.getWorksheets().get(sheetNo);
                        Cells cells = worksheet.getCells();
                        String sheetName = worksheet.getName();
                        String nameWithSheetName = sourceFile + BankStatementConstants.NAME_WITH_SHEET_NAME + sheetName;
                        Cell oaiCell1 = AsposeUtil.getCell(cells, "开户日期");
                        if (oaiCell1 != null) {//个人开户信息
                            if (excelFileName.endsWith(".xlsx")) {
                                personOai(oaiCell1, excelFileName, sheetNo, cells);
                            }

                            if (excelFileName.endsWith(".xls")) {
                                Cell tradeDateCell = AsposeUtil.getCell(cells, "发生额");
                                if (tradeDateCell != null) {//20240428新模板 个人，开户流水在一个模板里
                                    readBsAndOai(excelFileName, cells, sheetNo, nameWithSheetName);
                                } else {
                                    readOAI(cells);
                                }
                            }

                            templateIsExist = true;
                        }

                        if (!templateIsExist) {
                            Cell oaiCell2 = AsposeUtil.getCell(cells, "账户启用日期");
                            Cell oaiCell3 = AsposeUtil.getCell(cells, "合约建立日期");

                            if (oaiCell2 != null || oaiCell3 != null) {//公司开户信息
                                Cell tradeDateCell = AsposeUtil.getCell(cells, "交易日期");
                                if (tradeDateCell != null) {//20240416新模板 对公开户，流水在一个模板里
                                    readBsAndOai(excelFileName, cells, sheetNo, nameWithSheetName);
                                    templateIsExist = true;
                                } else {
                                    Cell tempCell = oaiCell3;
                                    if (oaiCell2 != null) {
                                        tempCell = oaiCell2;
                                        bsHasNotCard = true;//这种模板 开户+流水，但是流水里没有卡号，开户表里只能有一行
                                    }
                                    if (excelFileName.endsWith(".xls")) {
                                        companyOAIList = readCompanyOAIByXls(cells);
                                    } else {
                                        companyOai(tempCell, excelFileName, sheetNo, companyOAIList);
                                    }
                                    templateIsExist = true;
                                }

                            }
                        }

                        if (!templateIsExist && bsHasNotCard) {
                            Cell bsCell1 = AsposeUtil.getCell(cells, "交易行省市代码");
                            if (bsCell1 != null) {//这种模板 开户+流水，但是流水里没有卡号
                                //2023081615505557832-账号：24055401040001038
                                if (excelFileName.endsWith(".xls")) {
                                    readCompanyBSByXls(excelFileName, cells, companyOAIList.get(0));
                                } else {
                                    companyBs(bsCell1, excelFileName, sheetNo, companyOAIList.get(0));//这个表的时间长度较长
                                }
                                templateIsExist = true;
                            }
                        }

                        if (!templateIsExist) {
                            Cell bsCell2 = AsposeUtil.getCell(cells, "合约账户余额");
                            if (bsCell2 != null) {
                                companyBs(bsCell2, excelFileName, sheetNo, null);
                                templateIsExist = true;
                            }
                        }

                        if (!templateIsExist) {
                            Cell bsCell3 = AsposeUtil.getCell(cells, "子账号序号");
                            if (bsCell3 != null) {
                                Cell temp = AsposeUtil.getCell(cells, "交易日期");
                                if (temp != null) {
                                    personBs(bsCell3, excelFileName, sheetNo,nameWithSheetName);
                                    templateIsExist = true;
                                }
                            }
                        }

                        if (!templateIsExist) {
                            Cell bsCell4 = AsposeUtil.getCell(cells, "身份证号码");
                            if (bsCell4 != null) {
                                if (excelFileName.endsWith(".xlsx")) {
                                    personBs(bsCell4, excelFileName, sheetNo,nameWithSheetName);
                                }

                                if (excelFileName.endsWith(".xls")) {
                                    readABCCustomerBS(excelFileName, cells, null);
                                }

                                templateIsExist = true;
                            }
                        }

                        if (!templateIsExist) {
                            Cell flagCell = AsposeUtil.getCell(cells, "中国农业银行司法查询控制系统");
                            if (flagCell != null) {
                                Cell temp = AsposeUtil.getCell(cells, "交易日期");
                                if (temp != null) {
                                    personBs(temp, excelFileName, sheetNo,nameWithSheetName);
                                    templateIsExist = true;
                                }
                            }
                        }

                        if (!templateIsExist && (!IGNORE_SHEET_NAME.contains(sheetName))) {
                            throw new TemplateNotFindException(sourceFile + BankStatementConstants.NAME_WITH_SHEET_NAME + sheetName);
                        }
                    } catch (Exception e) {
                        importResultService.record(caseId, BANK_NAME, e);
                    }
                }
            } catch (Exception e) {
                importResultService.record(caseId, BANK_NAME, e);
            }
        }
    }

    private void readBsAndOai(String excelFileName, Cells cells,int sheetNo,String nameWithSheetName) {
        Cell cardCell = AsposeUtil.getCell(cells, "产品号/账号/客户号");
        if (cardCell == null) {
            cardCell = AsposeUtil.getCell(cells, "卡号");
            if(cardCell == null){
                throw new TemplateNotFindException(nameWithSheetName);
            }
        }
        String cardNumber = getCardNumber(cardCell,cells,nameWithSheetName);
        String cardHolderName = getCardHolderName(cells,nameWithSheetName);

        List<ABCCompanyInfoEntity> oaiList = new ArrayList<>();
        try (ExcelReader reader = EasyExcel.read(excelFileName).build()) {
            ReadSheet sheet = EasyExcel.readSheet(sheetNo)
                    .headRowNumber(cardCell.getRow()+1)
                    .head(ABCCompanyInfoEntity.class)
                    .registerReadListener(companyOAIReadListener(oaiList))
                    .build();
            reader.read(sheet);
        } catch (Exception e) {
            throw new ImportDataFailedException(e.getMessage(), nameWithSheetName);
        }


        Cell tradeDateCell = AsposeUtil.getCell(cells, "交易日期");
        if (tradeDateCell == null) {
            tradeDateCell = AsposeUtil.getCell(cells, "发生额");//20240428
            if(tradeDateCell == null){
                throw new TemplateNotFindException(nameWithSheetName);
            }
        }
        try (ExcelReader reader = EasyExcel.read(excelFileName).build()) {
            ReadSheet sheet = EasyExcel.readSheet(sheetNo)
                    .headRowNumber(tradeDateCell.getRow() + 1)
                    .head(ABCCompanyStatementEntity.class)
                    .registerReadListener(companyBsListenerFor20240416(cardNumber,cardHolderName,nameWithSheetName))
                    .build();
            reader.read(sheet);
        } catch (Exception e) {
            throw new ImportDataFailedException(e.getMessage(), nameWithSheetName);
        }

    }


    private String getCardNumber(Cell cardCell,Cells cells,String sourceFile) {

        int startRow = cardCell.getRow();
        int startCol = cardCell.getColumn();
        Cell cardNumberCell = cells.get(startRow + 1, startCol);
        if(null == cardNumberCell || StrUtil.isEmpty(cardNumberCell.getStringValue())){
            throw new ImportDataFailedException("读取卡号信息异常", sourceFile);
        }
        String cardNumber = cardNumberCell.getStringValue().trim();
        if(cardNumber.contains("/")){
            cardNumber = cardNumber.split("/")[0];
        }
        return cardNumber;
    }

    private String getCardHolderName(Cells cells,String sourceFile) {
        Cell cardCell = AsposeUtil.getCell(cells, "户名");
        if (cardCell == null) {
            throw new ImportDataFailedException("无法读取表头信息.", sourceFile);
        }
        int startRow = cardCell.getRow();
        int startCol = cardCell.getColumn();
        Cell cardNumberCell = cells.get(startRow + 1, startCol);
        if(null == cardNumberCell || StrUtil.isEmpty(cardNumberCell.getStringValue())){
            throw new ImportDataFailedException("读取户名信息异常", sourceFile);
        }
        String cardNumber = cardNumberCell.getStringValue().trim();
        if(cardNumber.contains("/")){
            cardNumber = cardNumber.split("/")[0];
        }
        return cardNumber;
    }


    void personBs(Cell bsCell, String excelFileName, int sheetNo,String nameWithSheetName){
        int headRowNum = bsCell.getRow() + 1;

        try (ExcelReader reader = EasyExcel.read(excelFileName).build()) {
            ReadSheet sheet = EasyExcel.readSheet(sheetNo)
                    .headRowNumber(headRowNum)
                    .head(ABCCustomerStatementEntity.class)
                    .registerReadListener(personBSReadListener(nameWithSheetName))
                    .build();
            reader.read(sheet);
        } catch (Exception e) {
            throw new ImportDataFailedException(e.getMessage(), excelFileName);
        }
    }

    private void companyBs(Cell tempCell, String excelFileName, int sheetNo,ABCCompanyInfoEntity oai) {
        String sourceFile = HelperUtil.getSourceFileName(excelFileName,BANK_NAME);
        int headRowNum = tempCell.getRow() + 1;
        try (ExcelReader reader = EasyExcel.read(excelFileName).build()) {
            ReadSheet sheet = EasyExcel.readSheet(sheetNo)
                    .headRowNumber(headRowNum)
                    .head(ABCCompanyStatementEntity.class)
                    .registerReadListener(companyBSReadListener(oai,sourceFile))
                    .build();
            reader.read(sheet);
        } catch (Exception e) {
            throw new ImportDataFailedException(e.getMessage(), excelFileName);
        }
    }


    private void companyOai(Cell tempCell, String excelFileName, int sheetNo,List<ABCCompanyInfoEntity> oaiList) {
        int headRowNum = tempCell.getRow() + 1;
        try (ExcelReader reader = EasyExcel.read(excelFileName).build()) {
            ReadSheet sheet = EasyExcel.readSheet(sheetNo)
                    .headRowNumber(headRowNum)
                    .head(ABCCompanyInfoEntity.class)
                    .registerReadListener(companyOAIReadListener(oaiList))
                    .build();
            reader.read(sheet);


        } catch (Exception e) {
            throw new ImportDataFailedException(e.getMessage(), excelFileName);
        }
    }

    private void personOai(Cell oaiCell1, String excelFileName,int sheetNo,Cells cells) {
        int headRowNum = oaiCell1.getRow() + 1;
        try (ExcelReader reader = EasyExcel.read(excelFileName).build()) {
            ReadSheet sheet = EasyExcel.readSheet(sheetNo)
                    .headRowNumber(headRowNum)
                    .head(ABCCustomerInfoEntity.class)
                    .registerReadListener(personOAIReadListener())
                    .build();
            reader.read(sheet);
        } catch (Exception e) {
            if(e instanceof OLE2NotOfficeXmlFileException){
                String sourceFile = HelperUtil.getSourceFileName(excelFileName,BANK_NAME);
                log.error("xlsx是手动改的后缀 而不是通过另存为的方式-"+sourceFile);
                try {
                    CellsWrapper<ABCCustomerInfoEntity> wrapper = new CellsWrapper<>(cells, ABCCustomerInfoEntity.class, false);
                    wrapper.doRead(-1);
                    List<ABCCustomerInfoEntity> entryList = wrapper.getEntryList();
                    esABCCustomerInfoMapper.insertBatch(entryList);
                } catch (Exception ex) {
                    throw new ImportDataFailedException(ex.getMessage(), sourceFile);
                }

            }else {
                throw new AnalyzeDataFailedException(e.getMessage(),e);
//                HelperUtil.recordErrorImportResult(e.getMessage(), sourceFile,sheetName,caseId,BANK_NAME);
            }
        }
    }



    public ReadListener<ABCCompanyStatementEntity> companyBsListenerFor20240416(String cardNumber,String cardHolderName,String sourceFile) {
        return new ReadListener<ABCCompanyStatementEntity>() {

            List<ABCCompanyStatementEntity> cacheList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);

            @Override
            public void invoke(ABCCompanyStatementEntity entity, AnalysisContext context) {
                // 4 交易日期
                String transDate = entity.getTransactionDate();
                try {
                    if (StrUtil.isNotEmpty(transDate)) { // 20211006
                        String transTime = entity.getTransactionTime();
                        if (StrUtil.isNotEmpty(transTime)) { // 14:15:07
                            if (transTime.length() > Constants.DATE_TIME_FULL_FORMAT_LENGTH) {
                                // 20131001145718000
                                transTime = transTime.substring(8, 14);
                                DateUtil.parse(transTime, "HHmmss");
                                entity.setTransactionTime(transTime);
                            } else { // 20081222 030411
                                if (transTime.contains(":")) {
                                    DateUtil.parse(transDate + " " + transTime, "yyyyMMdd HH:mm:ss");
                                } else {
                                    DateUtil.parse(transDate + transTime, "yyyyMMddHHmmss");
                                }
                            }
                        } else {
                            DateUtil.parse(transDate, "yyyyMMdd");
                        }
                    }
                } catch (Exception e) {
                    log.error(sourceFile+"解析交易日期异常", e);
                    Integer currentRowNum = context.getCurrentRowNum() + 1;
                    throw new AnalyzeDataFailedException("第"+ currentRowNum +"行解析交易日期异常", e, sourceFile);
                }
                entity.setId(IdUtil.objectId());
                entity.setCardNumber(cardNumber);
                entity.setCardHolderName(cardHolderName);
                entity.setSourceFile(sourceFile);
                cacheList.add(entity);
                if (cacheList.size() >= Constants.BATCH_SIZE) {
                    save();
                }
            }

            @Override
            public void doAfterAllAnalysed(AnalysisContext context) {
                if (!cacheList.isEmpty()) {
                    save();
                }
            }

            private void save() {
                esABCCompanyStatementMapper.insertBatch(cacheList);
                cacheList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
            }
        };
    }


    public ReadListener<ABCCustomerInfoEntity> personOAIReadListener() {
        return new ReadListener<ABCCustomerInfoEntity>() {

            List<ABCCustomerInfoEntity> cacheList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);

            @Override
            public void invoke(ABCCustomerInfoEntity entity, AnalysisContext context) {
                entity.setId(IdUtil.objectId());
                cacheList.add(entity);
                if (cacheList.size() >= Constants.BATCH_SIZE) {
                    save();
                }
            }

            @Override
            public void doAfterAllAnalysed(AnalysisContext context) {
                if (!cacheList.isEmpty()) {
                    save();
                }
            }

            private void save() {
                esABCCustomerInfoMapper.insertBatch(cacheList);
                cacheList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
            }
        };
    }

    public ReadListener<ABCCompanyInfoEntity> companyOAIReadListener(List<ABCCompanyInfoEntity> oaiList) {
        return new ReadListener<ABCCompanyInfoEntity>() {

            List<ABCCompanyInfoEntity> cacheList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);

            @Override
            public void invoke(ABCCompanyInfoEntity entity, AnalysisContext context) {
                String cardNumber = entity.getCardNumber();
                if(StrUtil.isEmpty(cardNumber)){
                    return;
                }
                if (HelperUtil.isContainsChinese(cardNumber)){
                    return;
                }
                cardNumber = entity.getCardNumber().trim();
                if(cardNumber.contains("/")){
                    cardNumber = cardNumber.split("/")[0];
                }

                if(StrUtil.isEmpty(entity.getIdCardNo())){
                    entity.setIdCardNo("");//做统计值是null不会被统计,空字符串可以
                }

                entity.setCardNumber(cardNumber);
                entity.setId(IdUtil.objectId());
                cacheList.add(entity);
                oaiList.add(entity);
                if (cacheList.size() >= Constants.BATCH_SIZE) {
                    save();
                }
            }

            @Override
            public void doAfterAllAnalysed(AnalysisContext context) {
                if (!cacheList.isEmpty()) {
                    save();
                }
            }

            private void save() {
                esABCCompanyInfoMapper.insertBatch(cacheList);
                cacheList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
            }
        };
    }

    public ReadListener<ABCCompanyStatementEntity> companyBSReadListener(ABCCompanyInfoEntity oai,String sourceFile) {
        return new ReadListener<ABCCompanyStatementEntity>() {

            List<ABCCompanyStatementEntity> cacheList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);

            @Override
            public void invoke(ABCCompanyStatementEntity entity, AnalysisContext context) {
                entity.setId(IdUtil.objectId());
                entity.setSourceFile(sourceFile);
                if(null != oai && StrUtil.isEmpty(entity.getCardNumber())){
                    entity.setCardNumber(oai.getCardNumber2());
                    entity.setCardHolderName(oai.getCardHolderName());
                }
                cacheList.add(entity);
                if (cacheList.size() >= Constants.BATCH_SIZE) {
                    save();
                }
            }

            @Override
            public void doAfterAllAnalysed(AnalysisContext context) {
                if (!cacheList.isEmpty()) {
                    save();
                }
            }

            private void save() {
                esABCCompanyStatementMapper.insertBatch(cacheList);
                cacheList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
            }
        };
    }

    public ReadListener<ABCCustomerStatementEntity> personBSReadListener(String sourceFile) {
        return new ReadListener<ABCCustomerStatementEntity>() {

            List<ABCCustomerStatementEntity> cacheList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);

            @Override
            public void invoke(ABCCustomerStatementEntity entity, AnalysisContext context) {
                entity.setId(IdUtil.objectId());
                entity.setSourceFile(sourceFile);
                cacheList.add(entity);
                if (cacheList.size() >= Constants.BATCH_SIZE) {
                    save();
                }
            }

            @Override
            public void doAfterAllAnalysed(AnalysisContext context) {
                if (!cacheList.isEmpty()) {
                    save();
                }
            }

            private void save() {
                esABCCustomerStatementMapper.insertBatch(cacheList);
                cacheList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
            }
        };
    }

    public void analyzeData(String caseId) {

        // 开户信息
        analyzeOAI(caseId);
        // 流水信息
        analyzeBS(caseId);
    }

    private void analyzeOAI(String caseId) {
        // 先分析公司数据
        analyzeCompanyOAI(caseId);
        analyzeCustomerOAI(caseId);
    }

    private void analyzeCustomerOAI(String caseId) {
        List<OpeningAccountInfo> oaiList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);

        List<ABCCustomerInfoEntity> entityList =
                HelperUtil.getEntityList(esABCCustomerInfoMapper, ABCCustomerInfoEntity.class);
        Set<String> uniqueKeySet = new HashSet();
        for (ABCCustomerInfoEntity entity : entityList) {

            OpeningAccountInfo oai = getOpeningAccountInfo( entity);
            String md5Id = HelperUtil.generateMD5Id4OAI(oai,caseId);
            //未导入数据内部去重
            if(HelperUtil.deduplication(md5Id,uniqueKeySet)){
                continue;
            }

            oai.setId(md5Id);
            oai.setCaseId(caseId);
            oaiList.add(oai);
            if (oaiList.size() >= Constants.BATCH_SIZE) {
                List<OpeningAccountInfo> dest = HelperUtil.getDest(oaiList);
                HelperUtil.batchSaveOAI2Es(dest, caseId);

                oaiList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
            }
        }
        uniqueKeySet.clear();
        if (!oaiList.isEmpty()) {
            List<OpeningAccountInfo> dest = HelperUtil.getDest(oaiList);
            HelperUtil.batchSaveOAI2Es(dest, caseId);
        }
    }

    private static OpeningAccountInfo getOpeningAccountInfo(ABCCustomerInfoEntity entity) {
        OpeningAccountInfo oai = new OpeningAccountInfo();
        oai.setBankName(BANK_NAME);

        String cardNumber = entity.getCardNumber();
        oai.setAccountNumber(cardNumber);
        oai.setIdNo(entity.getIdCardNo());

        oai.setName(entity.getCardHolderName());

        oai.setPhone(entity.getPhone());
        oai.setAddress(entity.getAddress());
        oai.setOpeningAccountDate(entity.getOpeningAccountDate());
        oai.setClosingDate(entity.getCloseDate());
        oai.setStatus(entity.getStatus());
        oai.setAccountOpeningInstitution(entity.getAccountOpeningInstitution());
        oai.setBalance(entity.getBalance());

        return oai;
    }

    private void analyzeCompanyOAI(String caseId) {

        List<OpeningAccountInfo> oaiList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);

        List<ABCCompanyInfoEntity> entityList =
                HelperUtil.getEntityList(esABCCompanyInfoMapper, ABCCompanyInfoEntity.class);
        Set<String> uniqueKeySet = new HashSet();
        for (ABCCompanyInfoEntity entity : entityList) {

            OpeningAccountInfo oai = getOpeningAccountInfo(entity);

            String md5Id = HelperUtil.generateMD5Id4OAI(oai,caseId);
            //未导入数据内部去重
            if(HelperUtil.deduplication(md5Id,uniqueKeySet)){
                continue;
            }

            oai.setId(md5Id);
            oai.setCaseId(caseId);
            oaiList.add(oai);
            if (oaiList.size() >= Constants.BATCH_SIZE) {
                List<OpeningAccountInfo> dest = HelperUtil.getDest(oaiList);
                HelperUtil.batchSaveOAI2Es(dest, caseId);

                oaiList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
            }
        }

        if (!oaiList.isEmpty()) {
            List<OpeningAccountInfo> dest = HelperUtil.getDest(oaiList);
            HelperUtil.batchSaveOAI2Es(dest, caseId);
        }
    }

    private static OpeningAccountInfo getOpeningAccountInfo(ABCCompanyInfoEntity entity) {
        OpeningAccountInfo oai = new OpeningAccountInfo();
        oai.setName(entity.getCardHolderName());
        oai.setBankName(BANK_NAME);
        oai.setAccountNumber(entity.getCardNumber());
        oai.setPhone(entity.getPhone());
        oai.setAddress(entity.getAddress());
        oai.setOpeningAccountDate(entity.getOpeningAccountDate());
        oai.setClosingDate(entity.getCloseDate());
        oai.setStatus(entity.getStatus());
        oai.setAccountOpeningInstitution(entity.getAccountOpeningInstitution());
        return oai;
    }

    private void analyzeBS(String caseId) {
        // 异步进行
        analyzeBSForCompany(caseId);
        analyzeBSForCustomer(caseId);
    }

    private void analyzeBSForCustomer(String caseId) {
        //去重
        Set<String> uniqueKeySet = new HashSet();
        List<PlateNumberInfo> plateNumberInfoList = new ArrayList<>();

        List<BankStatement> bsList = new ArrayList<>();


        Map<String, ABCCustomerInfoEntity> cache = new HashMap<>();

        List<OpeningAccountInfo> oaiList = HelperUtil.getEntityListV2(esOpeningAccountInfoMapper, OpeningAccountInfo.class, caseId);

        List<ABCCustomerInfoEntity> customerInfoList =
                HelperUtil.getEntityList(esABCCustomerInfoMapper, ABCCustomerInfoEntity.class);
        for (ABCCustomerInfoEntity info : customerInfoList) {
            cache.put(info.getIdCardNo(), info);
        }

        Map<String, List<ABCCustomerInfoEntity>> groupByCardNumber = customerInfoList.stream().filter(item ->StrUtil.isNotEmpty(item.getCardNumber()))
                .collect(Collectors.groupingBy(ABCCustomerInfoEntity::getCardNumber));
        Map<String, List<ABCCustomerInfoEntity>> groupByAccountNo = customerInfoList.stream().filter(item ->StrUtil.isNotEmpty(item.getAccountNo()))
                .collect(Collectors.groupingBy(ABCCustomerInfoEntity::getAccountNo));
        // 处理个人流水
        List<ABCCustomerStatementEntity> entityList =
                HelperUtil.getEntityList(esABCCustomerStatementMapper, ABCCustomerStatementEntity.class);
        entityList.stream().filter(item ->StrUtil.isNotEmpty(item.getAccountNo()))
                .collect(Collectors.groupingBy(ABCCustomerStatementEntity::getAccountNo));
        // 主要其实就是流水数据
        for (ABCCustomerStatementEntity entity : entityList) {
            String sourceFile = entity.getSourceFile();
            try {
                // 流水
                BankStatement bs = new BankStatement();
                bs.setBankName(BANK_NAME);
                String cardNumber = entity.getCardNumber();
                String accountNo = entity.getAccountNo();
                bs.setIdCardNo(entity.getIdCardNo());
                bs.setCardHolderName(entity.getCardHolderName());
                bs.setCardNumber(cardNumber);
                if (StrUtil.isEmpty(bs.getIdCardNo()) || StrUtil.isEmpty(bs.getCardHolderName())) {
                    if (groupByCardNumber.containsKey(cardNumber)) {
                        List<ABCCustomerInfoEntity> abcCustomerInfoEntities = groupByCardNumber.get(cardNumber);
                        ABCCustomerInfoEntity oai = abcCustomerInfoEntities.get(0);
                        bs.setIdCardNo(oai.getIdCardNo());
                        bs.setCardHolderName(oai.getCardHolderName());
                        bs.setCardNumber(StringUtils.isEmpty(oai.getCardNumber())?oai.getCardNumber2():oai.getCardNumber());
                    }

                    if(StrUtil.isEmpty(bs.getCardHolderName()) && StrUtil.isEmpty(entity.getCardHolderName()) && StrUtil.isNotEmpty(accountNo) ){
                        for (String oaiAccountNo : groupByAccountNo.keySet()) {
                            if(oaiAccountNo.contains(accountNo)){
                                List<ABCCustomerInfoEntity> abcCustomerInfoEntities = groupByAccountNo.get(oaiAccountNo);
                                ABCCustomerInfoEntity oai = abcCustomerInfoEntities.get(0);
                                bs.setIdCardNo(oai.getIdCardNo());
                                bs.setCardHolderName(oai.getCardHolderName());
                                bs.setCardNumber(StringUtils.isEmpty(oai.getCardNumber())?oai.getCardNumber2():oai.getCardNumber());
                                break;
                            }
                        }
                    }
                }


                ABCCustomerInfoEntity info = cache.getOrDefault(bs.getIdCardNo(), null);
                if (info != null) {
                    bs.setPhone(info.getPhone());
                    if (StringUtils.isEmpty(bs.getCardHolderName())) {
                        bs.setCardHolderName(info.getCardHolderName());
                        if (StringUtils.isEmpty(bs.getCardHolderName())) {
                            Optional<ABCCustomerInfoEntity> first = cache.values().stream().filter(item -> StrUtil.isNotEmpty(item.getCardNumber())).filter(cs -> cs.getCardNumber().equalsIgnoreCase(bs.getCardNumber())).findFirst();
                            first.ifPresent(abcCustomerInfoEntity -> bs.setCardHolderName(abcCustomerInfoEntity.getCardHolderName()));
                            first.ifPresent(abcCustomerInfoEntity -> bs.setIdCardNo(abcCustomerInfoEntity.getIdCardNo()));
                        }
                    }

                    if (StringUtils.isEmpty(bs.getCardNumber())) {
                        bs.setCardNumber(info.getCardNumber());
                        if (StringUtils.isEmpty(bs.getCardNumber())) {
                            Optional<ABCCustomerInfoEntity> first = cache.values().stream().filter(item -> StrUtil.isNotEmpty(item.getCardNumber())).filter(cs -> cs.getIdCardNo().equalsIgnoreCase(bs.getIdCardNo())).findFirst();
                            first.ifPresent(abcCustomerInfoEntity -> bs.setCardNumber(abcCustomerInfoEntity.getCardNumber()));
                        }
                    }


                    cache.put(info.getIdCardNo(), info);
                }

                if (StringUtils.isEmpty(bs.getIdCardNo()) && StringUtils.isNotEmpty(bs.getCardNumber())) {
                    Optional<OpeningAccountInfo> first = oaiList.stream().filter(cs -> StrUtil.isNotEmpty(cs.getAccountNumber())).filter(cs -> cs.getAccountNumber().equalsIgnoreCase(bs.getCardNumber())).findFirst();
                    first.ifPresent(abcCustomerInfoEntity -> bs.setIdCardNo(abcCustomerInfoEntity.getIdNo()));
                }
                // 1 货币类型
                // 人民币
                bs.setTransCurrencyType(Constants.CURRENCY_TYPE_CHINA);

                // 2 交易金额 可能是double或者int类型
                bs.setTransactionAmount(entity.getTransactionAmount());

                // 3 余额
                bs.setBalance(entity.getBalance());
                // 4 交易时间
                String transactionDate = entity.getTransactionDate();
                String transTime = entity.getTransactionTime();
                if (StrUtil.isNotEmpty(transactionDate)) {

                    if (StringUtils.isEmpty(transTime)) {
                        transTime = "000000";
                    }
                    if(!isInteger(transTime)){
                        transTime = "000000";
                    }

                    String transactionTime = transactionDate + transTime;
                    String format = "yyyyMMddHHmmss";
                    try {

                        if (transactionTime.length() > format.length()) {
                            transactionTime = transactionTime.substring(0, format.length());
                        }

                        bs.setTransactionTime(DateUtil.parse(transactionTime, format));
                    } catch (Exception e) {

                        String filename = entity.getSourceFile();

                        log.error("解析交易时间异常: {}{}", e.getMessage(), e,filename);
                        throw new AnalyzeDataFailedException(String.format("解析交易时间错误，无法将【%s】格式化为【%s】", transactionTime, format), e, sourceFile);
                    }
                }

                String counterpartyName = entity.getCounterpartyName();
                String counterpartyIdCardNo = entity.getCounterpartyIdCardNo();
                bs.setCounterpartIdCardNo(counterpartyIdCardNo);
                if (StrUtil.isNotEmpty(counterpartyName) && StrUtil.isEmpty(counterpartyIdCardNo)) {
                    List<ABCCustomerInfoEntity> rstList = cache.values().stream()
                        .filter(cs -> cs.getCardHolderName().equalsIgnoreCase(counterpartyName))
                        .collect(Collectors.toList());
                    if (!rstList.isEmpty()) {
                        ABCCustomerInfoEntity cs = rstList.get(0);
                        bs.setCounterpartIdCardNo(cs.getIdCardNo());
                    }
                }

                bs.setCounterpartyName(counterpartyName);
                bs.setCounterpartyBankName(entity.getCounterpartyBankName());
                bs.setCounterpartyAccount(entity.getCounterpartyAccount());
                bs.setTransactionInstitutions(entity.getTransactionInstitutions());
                bs.setSummary(entity.getSummary());
                bs.setTransRemark(entity.getTransRemark());
                bs.setTransChannel(entity.getTransChannel());

                String md5Id = HelperUtil.generateMD5Id(bs,caseId);
                //未导入数据内部去重
                if(HelperUtil.deduplication(md5Id,uniqueKeySet)){
                    continue;
                }

                bs.setSourceFile(entity.getSourceFile());
                bs.setId(md5Id);
                bs.setCaseId(caseId);
                try {
                    BeanUtils.beanAttributeValueTrim(bs);
                } catch (Exception e) {
                    e.printStackTrace();
                }
                bsList.add(bs);
                HelperUtil.extractPlateNumber(bs,plateNumberInfoList);


                if (bsList.size() >= Constants.BATCH_SIZE) {
                    List<BankStatement> dest = HelperUtil.getDest(bsList);
                    HelperUtil.batchInsert2Es(dest, caseId);

                    bsList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
                }
            } catch (Exception e) {
                importResultService.record(caseId, BANK_NAME, e, sourceFile);
            }
        }

        uniqueKeySet.clear();
        HelperUtil.batchInsertPlateNumber(plateNumberInfoList);

        // 保存数据库
        if (!bsList.isEmpty()) {
            List<BankStatement> dest = HelperUtil.getDest(bsList);
            HelperUtil.batchInsert2Es(dest, caseId);
        }
    }

    private void analyzeBSForCompany(String caseId) {
        List<BankStatement> bsList = new ArrayList<>();


        List<ABCCompanyStatementEntity> entityList =
                HelperUtil.getEntityList(esABCCompanyStatementMapper, ABCCompanyStatementEntity.class);


        List<ABCCompanyInfoEntity> companyInfoList =
                HelperUtil.getEntityList(esABCCompanyInfoMapper, ABCCompanyInfoEntity.class);
        Map<String, ABCCompanyInfoEntity> nameCompanyInfoMap = new HashMap<>();

        for (ABCCompanyInfoEntity entity : companyInfoList) {
            nameCompanyInfoMap.put(entity.getCardHolderName(), entity);
        }

        Map<String, List<ABCCompanyInfoEntity>> groupByContractNo = companyInfoList.stream()
                .filter(item ->StrUtil.isNotEmpty(item.getContractNo()))
                .collect(Collectors.groupingBy(ABCCompanyInfoEntity::getContractNo));


        Set<String> uniqueKeySet = new HashSet();
        List<PlateNumberInfo> plateNumberInfoList = new ArrayList<>();

        for (ABCCompanyStatementEntity entity : entityList) {
            String sourceFile = entity.getSourceFile();
            try {
                // 流水
                BankStatement bs = new BankStatement();
                bs.setBankName(BANK_NAME);

                String cardHolderName = entity.getCardHolderName();
                bs.setCardHolderName(cardHolderName);
                bs.setCardNumber(entity.getCardNumber());
                if (StrUtil.isEmpty(bs.getCardHolderName())) {
                    if (groupByContractNo.containsKey(entity.getContractNo())) {
                        List<ABCCompanyInfoEntity> abcCompanyInfoEntities = groupByContractNo.get(entity.getContractNo());
                        ABCCompanyInfoEntity companyInfo = abcCompanyInfoEntities.get(0);
                        bs.setCardHolderName(companyInfo.getCardHolderName());
                        String cardNumber = companyInfo.getCardNumber();
                        cardNumber = StrUtil.isEmpty(cardNumber) ? companyInfo.getCardNumber2() : cardNumber;
                        bs.setCardNumber(StrUtil.isEmpty(cardNumber) ? companyInfo.getContractNo() : cardNumber);
                        bs.setIdCardNo(companyInfo.getIdCardNo());
                    }
                }
                if (StrUtil.isEmpty(bs.getIdCardNo())) {
                    bs.setIdCardNo("");
                }
                ABCCompanyInfoEntity ci = nameCompanyInfoMap.getOrDefault(cardHolderName, null);
                if (ci != null) {
                    if (StrUtil.isEmpty(bs.getCardNumber())) {
                        bs.setCardNumber(ci.getCardNumber());
                    }
                    bs.setPhone(ci.getPhone());
                }
                // 1 货币类型
                // 人民币
                bs.setTransCurrencyType(Constants.CURRENCY_TYPE_CHINA);

                // 2 交易金额 可能是double或者int类型
                bs.setTransactionAmount(entity.getTransactionAmount());

                // 3 余额
                bs.setBalance(entity.getBalance());
                // 4 交易日期 20170314143047165086
                String transactionDate = entity.getTransactionDate();
                if (StrUtil.isNotEmpty(transactionDate)) {
                    // 如果包含-，则用yyyy-MM-dd来解析
                    String format;
                    if (transactionDate.contains("-")) {
                        format = "yyyy-MM-dd";
                    } else if (transactionDate.contains("/")) {
                        format = "yyyy/MM/dd";
                    } else {
                        format = "yyyyMMdd";
                    }

                    String transactionTime = entity.getTransactionTime();
                    // 如果不为空，则组合
                    if (StrUtil.isNotEmpty(transactionTime)) {
                        if (!isInteger(transactionTime)) {
                            transactionTime = "00:00:00";
                        }
                        // 还需要分情况考虑，一种是交易时间的长度超过了14位，另一种是交易时间的长度小于14位
                        if (transactionTime.length() > Constants.DATE_TIME_FULL_FORMAT_LENGTH) {
                            transactionTime = transactionTime.substring(
                                    Constants.DATE_FULL_FORMAT_LENGTH, Constants.DATE_TIME_FULL_FORMAT_LENGTH);
                        }

                        transactionDate += " " + transactionTime;
                        // 如果包含：，则用HH:mm:ss来解析
                        if (transactionTime.contains(":")) {
                            format = format + " HH:mm:ss";
                        } else {
                            format += " HHmmss";
                        }

                        try {
                            bs.setTransactionTime(DateUtil.parse(transactionDate, format));
                        } catch (Exception e) {
                            log.error("解析交易日期异常:{},{}", e.getMessage() + entity.getSourceFile(), e);
                            throw new AnalyzeDataFailedException(
                                    "解析交易日期失败，无法将【" + transactionDate + transactionTime + "】以格式【" + format + "】解析", e, entity.getSourceFile());
                        }
                    }
                } else {
                    // 一般情况下，交易日期不为空
                    // 若是为空，则交易日期和时间同时包含在交易时间内哦
                    String transactionTime = entity.getTransactionTime();
                    if (transactionTime != null) {

                        String format = "yyyyMMddHHmmss";

                        try {
                            // 需要截断 20170314 143047
                            int len = format.length();
                            if (transactionTime.length() > len) {
                                transactionTime = transactionTime.substring(0, len);
                            }
                            bs.setTransactionTime(DateUtil.parse(transactionTime, format));
                        } catch (Exception e) {
                            log.error("解析交易日期异常:{}", e.getMessage(), e);


                            throw new AnalyzeDataFailedException(
                                    "解析交易日期失败，无法将【" + transactionTime + "】以格式【" + format + "】解析", e, sourceFile);
                        }
                    }
                }

                String counterpartyName = entity.getCounterpartyName();
                bs.setCounterpartyName(counterpartyName);


                bs.setCounterpartyAccount(entity.getCounterpartyAccount());
                bs.setCounterpartyBankName(entity.getCounterpartyBankName());
                bs.setTransChannel(entity.getTransChannel());
                bs.setSummary(entity.getSummary());
                bs.setTransactionInstitutions(entity.getTransactionInstitutions());

                String md5Id = HelperUtil.generateMD5Id(bs, caseId);
                //未导入数据内部去重
                if (HelperUtil.deduplication(md5Id, uniqueKeySet)) {
                    continue;
                }

                bs.setSourceFile(entity.getSourceFile());
                bs.setId(md5Id);
                bs.setCaseId(caseId);
                try {
                    BeanUtils.beanAttributeValueTrim(bs);
                } catch (Exception e) {
                    e.printStackTrace();
                }
                bsList.add(bs);
                HelperUtil.extractPlateNumber(bs,plateNumberInfoList);


                if (bsList.size() >= Constants.BATCH_SIZE) {
                    List<BankStatement> dest = HelperUtil.getDest(bsList);
                    HelperUtil.batchInsert2Es(dest, caseId);

                    bsList.clear();
                    bsList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
                }
            } catch (Exception e) {
                importResultService.record(caseId, BANK_NAME, e, sourceFile);
            }
        }

        uniqueKeySet.clear();
        HelperUtil.batchInsertPlateNumber(plateNumberInfoList);
        // 保存数据库
        if (!bsList.isEmpty()) {
            List<BankStatement> dest = HelperUtil.getDest(bsList);
            HelperUtil.batchInsert2Es(dest, caseId);
        }
    }

    public static boolean isInteger(String str) {
        Pattern pattern = Pattern.compile("^[-\\+]?[\\d]*$");
        return pattern.matcher(str).matches();
    }

    private List<ABCCustomerInfoEntity> readOAI( Cells cells) throws Exception {
            CellsWrapper<ABCCustomerInfoEntity> wrapper = new CellsWrapper<>(cells, ABCCustomerInfoEntity.class, false);
            wrapper.doRead(-1);
            List<ABCCustomerInfoEntity> entryList = wrapper.getEntryList();
            esABCCustomerInfoMapper.insertBatch(entryList);
            return entryList;
    }

    private List<ABCCompanyInfoEntity> readCompanyOAIByXls(Cells cells) throws Exception {
            CellsWrapper<ABCCompanyInfoEntity> wrapper = new CellsWrapper<>(cells, ABCCompanyInfoEntity.class, false);
            wrapper.doRead(-1);
            List<ABCCompanyInfoEntity> entryList = wrapper.getEntryList();
            esABCCompanyInfoMapper.insertBatch(entryList);
            return entryList;
    }

    private void readABCCustomerBS(String excelFileName, Cells cells, List<ABCCustomerInfoEntity> oaiList) throws Exception {
        String sourceFile = HelperUtil.getSourceFileName(excelFileName,BANK_NAME);

        // 读个人流水
        try {
            CellsWrapper<ABCCustomerStatementEntity> wrapper = new CellsWrapper<>(cells, ABCCustomerStatementEntity.class, true);
            wrapper.doRead(-1);
            List<ABCCustomerStatementEntity> entryList = wrapper.getEntryList();
            Map<String, List<ABCCustomerInfoEntity>> groupByAccountNo = new HashMap<>();
            if(CollectionUtil.isNotEmpty(oaiList)){
                groupByAccountNo = oaiList.stream().collect(Collectors.groupingBy(ABCCustomerInfoEntity::getAccountNo));
            }
            for (ABCCustomerStatementEntity bs : entryList) {
                bs.setId(IdUtil.objectId());
                bs.setSourceFile(sourceFile);
                String accountNo = bs.getAccountNo();
                if(StrUtil.isNotEmpty(accountNo)){
                    if (groupByAccountNo.containsKey(accountNo)) {
                        List<ABCCustomerInfoEntity> tempList = groupByAccountNo.get(accountNo);
                        ABCCustomerInfoEntity oai = tempList.get(0);
                        bs.setCardHolderName(oai.getCardHolderName());
                        bs.setCardNumber(oai.getCardNumber());
                    }
                }
            }

            //save to es
            esABCCustomerStatementMapper.insertBatch(entryList);

        } catch (Exception e) {

            if (e instanceof ExcelAnalysisStopException) {
                return;
            }

            log.error("readABCCustomerBS:读取文件异常:{}", e.getMessage(), e);
            throw new ImportDataFailedException(e.getMessage(), excelFileName);
        }
    }

    private void readCompanyBSByXls(String excelFileName, Cells cells,ABCCompanyInfoEntity oai) throws Exception {
        String sourceFile = HelperUtil.getSourceFileName(excelFileName,BANK_NAME);

        try {

            CellsWrapper<ABCCompanyStatementEntity> wrapper = new CellsWrapper<>(cells, ABCCompanyStatementEntity.class, true);
            wrapper.doRead(-1);
            String cardHolderName = oai.getCardHolderName();
            String cardNumber = oai.getCardNumber();
            if(StrUtil.isEmpty(cardNumber)){
                cardNumber = oai.getCardNumber2();
            }else {
                if(cardNumber.length() > 17){
                    cardNumber = oai.getCardNumber2();
                }
            }
            List<ABCCompanyStatementEntity> entryList = wrapper.getEntryList();
            //加ID
            for (ABCCompanyStatementEntity bs : entryList) {
                bs.setId(IdUtil.objectId());
                bs.setSourceFile(sourceFile);
                bs.setCardNumber(cardNumber);
                bs.setCardHolderName(cardHolderName);
            }

            //save to es
            esABCCompanyStatementMapper.insertBatch(entryList);

        } catch (Exception e) {
            log.error("readABCCompanyBS:读取文件异常:{}", e.getMessage(), e);
            throw new ImportDataFailedException(e.getMessage(), excelFileName);
        }
    }

}