case-analysis-service/cas-system/src/main/java/com/inscloudtech/bankStatementAnalysis/helper/ICBCDataAnalysisHelper.java

package com.inscloudtech.bankStatementAnalysis.helper;

import cn.hutool.core.date.DateUtil;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSON;
import cn.hutool.json.JSONObject;
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelReader;
import com.alibaba.excel.read.metadata.ReadSheet;
import com.alibaba.excel.util.ListUtils;
import com.alibaba.excel.util.StringUtils;
import com.aspose.cells.Cell;
import com.aspose.cells.Cells;
import com.aspose.cells.Workbook;
import com.aspose.cells.Worksheet;

import com.inscloudtech.bankStatementAnalysis.domain.HeadField;
import com.inscloudtech.bankStatementAnalysis.helper.HelperUtil;
import com.inscloudtech.bankStatementAnalysis.service.ImportService;
import com.inscloudtech.common.constant.BankStatementConstants;
import com.inscloudtech.common.constant.Constants;
import com.inscloudtech.common.exception.dc.AnalyzeDataFailedException;
import com.inscloudtech.common.exception.dc.ImportDataFailedException;
import com.inscloudtech.common.exception.dc.TemplateNotFindException;
import com.inscloudtech.common.utils.bean.BeanUtils;
import com.inscloudtech.common.utils.file.FileUtils;
import com.inscloudtech.datacenter.domain.PlateNumberInfo;
import com.inscloudtech.bankStatementAnalysis.mapper.EsICBCBankStatementMapper;
import com.inscloudtech.bankStatementAnalysis.mapper.EsICBCCreditCardBankStatementMapper;
import com.inscloudtech.datacenter.service.ImportResultService;
import com.inscloudtech.datacenter.domain.BankStatement;
import com.inscloudtech.datacenter.domain.OpeningAccountInfo;

import com.inscloudtech.bankStatementAnalysis.domain.entity.ICBCBankStatementEntry;
import com.inscloudtech.bankStatementAnalysis.domain.entity.ICBCCreditCardBankStatementEntry;

import com.inscloudtech.bankStatementAnalysis.util.AsposeUtil;

import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;

import java.io.File;
import java.math.BigDecimal;
import java.util.*;

/**
 * 工商银行数据分析
 */
@Slf4j
@RequiredArgsConstructor
@Component
public class ICBCDataAnalysisHelper {

    private final ImportService importService;
    private final ImportResultService importResultService;
    private final EsICBCBankStatementMapper bankStatementMapper;
    private final EsICBCCreditCardBankStatementMapper creditCardBankStatementMapper;
    private final static String BANK_NAME = "工商银行";


    public void importData(File dir,String caseId) throws Exception {
        List<File> fileList = FileUtil.loopFiles(dir);
        if (fileList == null) {
            return;
        }


        List<File> excelFileList = HelperUtil.getExcelFile(fileList);


//        List<File> pdfFileList = fileList.stream()
//                .filter(file -> FileUtil.extName(file).equalsIgnoreCase("pdf"))
//                .collect(Collectors.toList());
//        Map<String, List<File>> groupByPath = excelFileList.stream().collect(Collectors.groupingBy(File::getAbsolutePath));
//        for (File file : pdfFileList) {
//            String excelFilename = AsposeUtil.pdf2ExcelV6(file.getAbsolutePath());
//            if(groupByPath.containsKey(excelFilename)){
//                continue;
//            }
//            excelFileList.add(new File(excelFilename));
//        }

        for (File f : excelFileList) {
            try {
                String absolutePath = f.getAbsolutePath();
                Workbook wb = new Workbook(absolutePath);
                int count = wb.getWorksheets().getCount();
                String sourceFile = HelperUtil.getSourceFileName(absolutePath, BANK_NAME);

                for (int sheetNo = 0; sheetNo < count; sheetNo++) {
                    try {
                        Worksheet worksheet = wb.getWorksheets().get(sheetNo);
                        Cells cells = worksheet.getCells();
                        Cell contentCell = cells.get(0, 0);
                        if (contentCell.getValue() == null) {//空白excel
                            continue;
                        }
                        String worksheetName = worksheet.getName();
                        Cell cell420240418 = AsposeUtil.getCell(cells, "入帐日期");
                        Cell khCell = AsposeUtil.getCell(cells, "开户日期");
                        Cell tradeCell = AsposeUtil.getCell(cells, "交易时间戳");
                        Cell khhzh = AsposeUtil.getCell(cells, "卡号或帐号");
                        String withSheetName = sourceFile + BankStatementConstants.NAME_WITH_SHEET_NAME + worksheetName;
                        /**20240418新增模板*/
                        if (cell420240418 != null) {
                            readBSFor20240418(cells,absolutePath, sheetNo,withSheetName);
                        }else if (khCell != null) {
                            // 判断是否是开户信息
                            int rowNumber = khCell.getRow() + 1;
                            importService.readOAIData(absolutePath, rowNumber,
                                    sheetNo, caseId, BANK_NAME, withSheetName);
                        } else if (tradeCell != null) { // 流水信息
                            readBS(absolutePath, sheetNo,withSheetName);
                        } else if (khhzh != null) { // 开户
                            importService.readOAIData(absolutePath, khhzh.getRow() + 1,
                                    sheetNo, caseId, BANK_NAME, withSheetName);
                        } else {
                            throw new TemplateNotFindException(withSheetName);
                        }
                    } catch (Exception e) {
                        importResultService.record(caseId, BANK_NAME, e);
                    }
                }
            } catch (Exception e) {
                importResultService.record(caseId, BANK_NAME, e);
            }
        }
    }

    private void readBSFor20240418(Cells cells ,String absolutePath, int sheetNo,String sourceFile) {
//        int startRow = dateCell.getRow();
//        Cell cardHolderNameCell = cells.get(startRow - 1, 1);
//        if(null == cardHolderNameCell || StrUtil.isEmpty(cardHolderNameCell.getStringValue())){
//            throw new ImportDataFailedException("读取持卡人信息异常", sourceFile);
//        }


        List<JSONObject> maps = new LinkedList<>();
        int maxRow = cells.getMaxRow();
        List<HeadField> headFields = new ArrayList<>();
        headFields.add(new HeadField("transactionAmount","发生额"));
        headFields.add(new HeadField("balance","余额"));
        headFields.add(new HeadField("accountNumber","商密二级,帐号"));
        for(int i = 0; i < maxRow; i++){
            Cell dateCell = cells.get(i, 2);
            String stringValue = dateCell.getStringValue();
            if (StrUtil.isEmpty(stringValue)) {
                continue;
            }
            if(stringValue.equals("入帐日期")){
                int startRow = dateCell.getRow();
                Cell cardHolderNameCell = cells.get(startRow - 1, 1);
                if(null == cardHolderNameCell || StrUtil.isEmpty(cardHolderNameCell.getStringValue())){
                    throw new ImportDataFailedException("读取持卡人信息异常", sourceFile);
                }
                String cardHolderName = cardHolderNameCell.getStringValue().trim();
                JSONObject jsonObject = new JSONObject();
                jsonObject.putOnce("startRow",i);
                jsonObject.putOnce("cardHolderName",cardHolderName);
                maps.add(jsonObject);
            }
        }
        int size = maps.size();
        for(int i = 0; i < size; i++){
            JSONObject jsonObject = maps.get(i);
            String cardHolderName = jsonObject.getStr("cardHolderName");
            int headRowNumber = jsonObject.getInt("startRow") + 1;
            int endRow = maxRow -1;
            if(i + 1 < size){
                JSONObject nextRow = maps.get(i + 1);
                endRow = nextRow.getInt("startRow") - 1;
            }

            if(endRow - headRowNumber <= 1){
                continue;
            }
            importService.readMultiplePersonAndMultipleHeadBankStatement(absolutePath,headRowNumber,endRow,sheetNo,
                ICBCBankStatementEntry.class,bankStatementMapper,cardHolderName,sourceFile,headFields);
        }


    }





    public void analyzeIData(String caseId) {
        // 储蓄卡流水
        analyzeSavingsCardBS(caseId);
        // 信用卡流水
        analyzeCreditCardBS(caseId);
    }

    private void analyzeCreditCardBS(String caseId) {
        List<BankStatement> bsList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);

        List<ICBCCreditCardBankStatementEntry> entityList =
                HelperUtil.getEntityList(creditCardBankStatementMapper, ICBCCreditCardBankStatementEntry.class);
        List<OpeningAccountInfo> oaiData = importService.getOAIData(caseId, BANK_NAME);
        Map<String, OpeningAccountInfo> nameAccMap = new HashMap<>();
        Map<String, OpeningAccountInfo> cardNumberAccMap = new HashMap<>();
        Map<String, OpeningAccountInfo> account2CardNumberMap = new HashMap<>();

        for (OpeningAccountInfo entry : oaiData) {
            if (StrUtil.isNotEmpty(entry.getName())) {
                nameAccMap.put(entry.getName(), entry);
            }

            if (StrUtil.isNotEmpty(entry.getAccountNumber())) {
                cardNumberAccMap.put(entry.getAccountNumber(), entry);
            }

            if (StrUtil.isNotEmpty(entry.getAccount2CardNumber())) {
                account2CardNumberMap.put(entry.getAccount2CardNumber(), entry);
            }

        }

        //去重
        Set<String> uniqueKeySet = new HashSet();
        List<PlateNumberInfo> plateNumberInfoList = new ArrayList<>();

        for (ICBCCreditCardBankStatementEntry entry : entityList) {
            String sourceFile = entry.getSourceFile();
            try {
                BankStatement bs = new BankStatement();
                bs.setBankName(BANK_NAME);
                bs.setCardNumber(entry.getCardNumber());
                bs.setCardHolderName(entry.getCardHolderName());
                if (StrUtil.isEmpty(bs.getCardHolderName())) {
                    // 从开户信息中获取数据
                    OpeningAccountInfo acc = cardNumberAccMap.getOrDefault(entry.getCardNumber(), null);
                    if (acc != null) {
                        bs.setCardHolderName(acc.getName());
                        bs.setIdCardNo(acc.getIdNo());
                    }else {
                        acc = account2CardNumberMap.getOrDefault(entry.getCardNumber(), null);
                        if (acc != null) {
                            bs.setCardHolderName(acc.getName());
                            bs.setIdCardNo(acc.getIdNo());
                        }
                    }
                }else {
                    // 从开户信息中获取数据
                    OpeningAccountInfo acc = nameAccMap.getOrDefault(entry.getCardHolderName(), null);
                    if (acc != null) {
                        bs.setIdCardNo(acc.getIdNo());
                    }
                }

                bs.setCounterpartyName(entry.getCounterpartyName());
                bs.setCounterpartIdCardNo(null);
                bs.setCounterpartyAccount(entry.getCounterpartyAccount());
                bs.setSummary(entry.getSummary());
                bs.setTransRemark(entry.getTransRemark());
                bs.setTransCurrencyType(Constants.CURRENCY_TYPE_CHINA);
                bs.setTransactionInstitutions(entry.getTransactionInstitutions());
                // 交易金额
                String loanFlag = entry.getLoanFlag();
                if (StrUtil.isNotEmpty(loanFlag)) {
                    if (Objects.equals("借", loanFlag)) {
                        bs.setTransactionAmount(BigDecimal.ZERO.subtract(entry.getTransactionAmount()));
                    } else {
                        bs.setTransactionAmount(entry.getTransactionAmount());
                    }
                }
                // 交易时间 	2015-08-29-13.16.32.670138
                String transactionTime = entry.getTransactionTime();
                String pattern = "yyyy-MM-dd-HH.mm.ss";
                int len = pattern.length();
                transactionTime = transactionTime.substring(0, len);

                try {
                    bs.setTransactionTime(DateUtil.parse(transactionTime, pattern));
                } catch (Exception e) {
                    log.error("解析时间出错", e);

                    throw new AnalyzeDataFailedException("解析时间出错: " + e.getMessage(), e, sourceFile);
                }
                // 信用卡余额为0
                bs.setBalance(entry.getUpdateBalance());
                bs.setRemark("信用卡");

                String md5Id = HelperUtil.generateMD5Id(bs,caseId);
                //未导入数据内部去重
                if(HelperUtil.deduplication(md5Id,uniqueKeySet)){
                    continue;
                }

                bs.setId(md5Id);
                bs.setCaseId(caseId);
                try {
                    BeanUtils.beanAttributeValueTrim(bs);
                } catch (Exception e) {
                    e.printStackTrace();
                }
                bsList.add(bs);
                HelperUtil.extractPlateNumber(bs,plateNumberInfoList);

                if (bsList.size() >= Constants.BATCH_SIZE) {
                    List<BankStatement> dest = HelperUtil.getDest(bsList);
                    HelperUtil.batchInsert2Es(dest, caseId);
                    bsList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
                }
            } catch (Exception e) {
                importResultService.record(caseId, BANK_NAME, e, sourceFile);
            }
        }

        uniqueKeySet.clear();
        HelperUtil.batchInsertPlateNumber(plateNumberInfoList);

        if (!bsList.isEmpty()) {
            List<BankStatement> dest = HelperUtil.getDest(bsList);
            HelperUtil.batchInsert2Es(dest, caseId);
        }
    }

    private void analyzeSavingsCardBS(String caseId) {

        List<BankStatement> bsList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);

        List<ICBCBankStatementEntry> entityList =
                HelperUtil.getEntityList(bankStatementMapper, ICBCBankStatementEntry.class);

//        Map<String, List<OpeningAccountInfo>> oaiMap = importService.getOAIMap(caseId, BANK_NAME);
        List<OpeningAccountInfo> oaiData = importService.getOAIData(caseId, BANK_NAME);
        Map<String, OpeningAccountInfo> nameAccMap = new HashMap<>();
        Map<String, OpeningAccountInfo> cardNumberAccMap = new HashMap<>();
        Map<String, OpeningAccountInfo> accountNumberAccMap = new HashMap<>();
        Map<String, OpeningAccountInfo> account2CardNumberMap = new HashMap<>();

        for (OpeningAccountInfo entry : oaiData) {
            if (StrUtil.isNotEmpty(entry.getName())) {
                nameAccMap.put(entry.getName(), entry);
            }

            if (StrUtil.isNotEmpty(entry.getAccountNumber())) {
                cardNumberAccMap.put(entry.getAccountNumber(), entry);
            }

            if (StrUtil.isNotEmpty(entry.getCustomerId())) {
                accountNumberAccMap.put(entry.getCustomerId(), entry);
            }
            if (StrUtil.isNotEmpty(entry.getAccount2CardNumber())) {
                account2CardNumberMap.put(entry.getAccount2CardNumber(), entry);
            }

        }

        Set<String> uniqueKeySet = new HashSet();
        List<PlateNumberInfo> plateNumberInfoList = new ArrayList<>();

        for (ICBCBankStatementEntry entry : entityList) {
            String sourceFile = entry.getSourceFile();
            try {
                String cardNumber = entry.getCardNumber();
                String accountNumber = entry.getAccountNumber();
                if (StrUtil.isEmpty(entry.getCardHolderName())) {
                    if (cardNumberAccMap.containsKey(cardNumber)) {
                        OpeningAccountInfo info = cardNumberAccMap.get(cardNumber);
                        entry.setCardHolderName(info.getName());
                        entry.setIdCardNo(info.getIdNo());
                    } else if (cardNumberAccMap.containsKey(accountNumber)) {
                        OpeningAccountInfo info = cardNumberAccMap.get(accountNumber);
                        entry.setCardHolderName(info.getName());
                        entry.setIdCardNo(info.getIdNo());
                    } else if(accountNumberAccMap.containsKey(cardNumber)){
                        OpeningAccountInfo info = accountNumberAccMap.get(cardNumber);
                        entry.setCardHolderName(info.getName());
                        entry.setIdCardNo(info.getIdNo());

                    } else if(accountNumberAccMap.containsKey(accountNumber)){
                        OpeningAccountInfo info = accountNumberAccMap.get(accountNumber);
                        entry.setCardHolderName(info.getName());
                        entry.setIdCardNo(info.getIdNo());

                    } else if(account2CardNumberMap.containsKey(accountNumber)){
                        OpeningAccountInfo info = account2CardNumberMap.get(accountNumber);
                        entry.setCardHolderName(info.getName());
                        entry.setIdCardNo(info.getIdNo());
                    }else if(account2CardNumberMap.containsKey(cardNumber)){
                        OpeningAccountInfo info = account2CardNumberMap.get(cardNumber);
                        entry.setCardHolderName(info.getName());
                        entry.setIdCardNo(info.getIdNo());
                    }

                }else {
                    OpeningAccountInfo acc = nameAccMap.getOrDefault(entry.getCardHolderName(), null);
                    if (acc != null) {
                        entry.setIdCardNo(acc.getIdNo());
                    }
                }


                BankStatement bs = new BankStatement();
                bs.setBankName(BANK_NAME);

                cardNumber = StrUtil.isEmpty(cardNumber)? accountNumber : cardNumber;

                bs.setCardNumber(cardNumber);

                bs.setCardHolderName(entry.getCardHolderName());
                bs.setIdCardNo(entry.getIdCardNo() == null?"":entry.getIdCardNo());

                String counterpartName = entry.getCounterpartName();
                bs.setCounterpartyName(counterpartName);



                bs.setCounterpartyAccount(entry.getCounterpartyAccount());
                bs.setCounterpartyBankName(entry.getCounterpartyBankName());
                bs.setTransRemark(entry.getTransRemark());
                bs.setTransCurrencyType(Constants.CURRENCY_TYPE_CHINA);
                bs.setTransactionInstitutions(entry.getTransactionInstitutions());
                // 交易金额
                String loanFlag = entry.getLoanFlag();
                try {
                    if (StrUtil.isNotEmpty(loanFlag) && StrUtil.isNotEmpty(entry.getTransactionAmount())) {
                        BigDecimal transactionAmount = new BigDecimal(entry.getTransactionAmount());
                        if (transactionAmount.compareTo(BigDecimal.ZERO) < 0) {
                            transactionAmount = BigDecimal.ZERO.subtract(transactionAmount);
                        }

                        if (Objects.equals("借", loanFlag)) {
                            bs.setTransactionAmount(BigDecimal.ZERO.subtract( new BigDecimal(entry.getTransactionAmount())));
                        } else {
                            bs.setTransactionAmount(transactionAmount);
                        }
                    }
                }catch (Exception e){
                    throw new AnalyzeDataFailedException(
                            StrUtil.format("金额, 无法将字符串【{}】化为金额。", entry.getTransactionAmount()), e, sourceFile);
                }


// 交易时间 	2015-08-29-13.16.32.670138
                String transactionTime = entry.getTransactionTime();
                String transactionDate = entry.getTransactionDate();
                try {
                    if(StrUtil.isNotEmpty(transactionDate)){
                        String format = "yyyy-MM-ddHH.mm.ss";
                        if(!transactionDate.contains("-")){
                            format = "yyyyMMddHH.mm.ss";
                        }
                        bs.setTransactionTime(DateUtil.parse(transactionDate + transactionTime, format));
                    }else if(StrUtil.isNotEmpty(transactionTime) && transactionTime.contains(".")){
                        String pattern = "yyyy-MM-dd-HH.mm.ss";
                        int len = pattern.length();
                        transactionTime = transactionTime.substring(0, len);
                        bs.setTransactionTime(DateUtil.parse(transactionTime, pattern));
                    }else {

                    }

                } catch (Exception e) {
                    throw new AnalyzeDataFailedException(StrUtil.format("解析时间失败，时间：{}", transactionTime), e,entry.getSourceFile());
                }

                bs.setBalance(new BigDecimal(entry.getBalance()));
                bs.setSummary(entry.getSummary());
                bs.setTransChannel(entry.getTransChannel());

                String md5Id = HelperUtil.generateMD5Id(bs,caseId);
                //未导入数据内部去重
                if(HelperUtil.deduplication(md5Id,uniqueKeySet)){
                    continue;
                }

                bs.setSourceFile(entry.getSourceFile());
                bs.setId(md5Id);
                bs.setCaseId(caseId);
                try {
                    BeanUtils.beanAttributeValueTrim(bs);
                } catch (Exception e) {
                    e.printStackTrace();
                }
                bsList.add(bs);
                HelperUtil.extractPlateNumber(bs,plateNumberInfoList);


                if (bsList.size() >= Constants.BATCH_SIZE) {
                    List<BankStatement> dest = HelperUtil.getDest(bsList);
                    HelperUtil.batchInsert2Es(dest, caseId);
                    bsList = ListUtils.newArrayListWithExpectedSize(Constants.BATCH_SIZE);
                }
            } catch (Exception e) {
                importResultService.record(caseId, BANK_NAME, e, sourceFile);
            }
        }

        uniqueKeySet.clear();
        HelperUtil.batchInsertPlateNumber(plateNumberInfoList);

        if (!bsList.isEmpty()) {
            List<BankStatement> dest = HelperUtil.getDest(bsList);
            HelperUtil.batchInsert2Es(dest, caseId);
        }
    }



    private void readBS(String excelFileName, int sheetNo,String sourceFile) {

        try {
            // 分两种流水，一种是信用卡流水，一种是储蓄卡流水
            int type = getBSType(excelFileName, sheetNo);
            if (type == Constants.ICBC_BS_TYPE_CREDIT_CARD) { // 信用卡流水
                try (ExcelReader reader = EasyExcel.read(excelFileName).build()) {
                    ReadSheet sheet = EasyExcel.readSheet(sheetNo)
                            .head(ICBCCreditCardBankStatementEntry.class)
                            .registerReadListener(HelperUtil.getReadListener(
                                    creditCardBankStatementMapper,
                                    ICBCCreditCardBankStatementEntry.class,
                                    "",sourceFile))
                            .build();
                    reader.read(sheet);


                } catch (Exception e) {
                    log.error("读取excel文件失败", e);
                    throw new ImportDataFailedException(e.getMessage(), excelFileName);
                }
            } else if (type == Constants.ICBC_BS_TYPE_DEBIT_CARD) { // 储蓄卡流水
                try (ExcelReader reader = EasyExcel.read(excelFileName).build()) {
                    ReadSheet sheet = EasyExcel.readSheet(sheetNo)
                            .head(ICBCBankStatementEntry.class)
                            .registerReadListener(HelperUtil.getReadListener(
                                    bankStatementMapper, ICBCBankStatementEntry.class, "",sourceFile))
                            .build();
                    reader.read(sheet);


                } catch (Exception e) {
                    log.error("读取excel文件失败", e);
                    throw new ImportDataFailedException(e.getMessage(), excelFileName);
                }
            }
        } catch (Exception e) {
            log.error("读取excel文件失败", e);
            throw new ImportDataFailedException(e.getMessage(), excelFileName);
        }
    }

    private int getBSType(String excelFilename, int sheetNo) {

        File file = new File(excelFilename);
        Cell cell = AsposeUtil.getCell(file, sheetNo, "更新后余额");
        if (cell != null) {
            return Constants.ICBC_BS_TYPE_CREDIT_CARD;
        } else {
            Cell c = AsposeUtil.getCell(file, sheetNo, "余额");
            if (c != null) {
                return Constants.ICBC_BS_TYPE_DEBIT_CARD;
            }
        }

        return -1;
    }

}