createmodel-java/src/main/java/ieven/server/webapp/domain/file/FileService.java

//
// Source code recreated from a .class file by IntelliJ IDEA
// (powered by FernFlower decompiler)
//

package ieven.server.webapp.domain.file;

import cn.hutool.core.date.DateUtil;
import cn.hutool.core.io.IoUtil;
import cn.hutool.core.util.IdUtil;
import com.mongodb.client.gridfs.model.GridFSFile;
import com.mongodb.client.result.DeleteResult;
import ieven.server.webapp.domain.IdInput;
import ieven.server.webapp.domain.data.DataMap;
import ieven.server.webapp.domain.data.Fields;
import ieven.server.webapp.domain.model.Model;
import ieven.server.webapp.infrastructure.wrapper.Mapped;
import ieven.server.webapp.service.MongoExcelService;
import ieven.server.webapp.util.excel.ExcelXlsReader;
import ieven.server.webapp.util.excel.ExcelXlsxReader;
import ieven.server.webapp.util.excel.PublicStatic;
import info.monitorenter.cpdetector.io.*;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.apache.commons.lang3.StringUtils;
import org.bson.types.ObjectId;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.data.domain.Sort;
import org.springframework.data.domain.Sort.Order;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update;
import org.springframework.data.mongodb.gridfs.GridFsOperations;
import org.springframework.data.mongodb.gridfs.GridFsResource;
import org.springframework.data.mongodb.gridfs.GridFsTemplate;
import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.annotation.AsyncResult;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.stereotype.Service;

import java.io.*;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.stream.Collectors;

@Service
@EnableAsync
@Slf4j
public class FileService {
    String[] order1 = {"反诈数据-订单明细", "治安数据-订单明细","经侦数据-订单明细","网黑数据-订单明细", "反诈治安-交易明细", "反诈治安-账户透视", "反诈治安-对手透视"};
    String[] order2 = {
            "五联单-注册信息",
            "五联单-登录日志",
            "五联单-交易记录",
            "五联单-账户明细",
            "五联单-转账明细",
            "五联单-转账分析",
            "五联单-账户透视",
            "五联单-对手透视",
            "五联单-IP地址分析",
            "五联单-收货地址分析"
    };
    String[] order3 = {"财付通-注册信息", "财付通-订单明细1","财付通-订单明细2", "财付通-交易明细", "财付通-账户透视", "财付通-对手透视", "财付通-手机明细"};

    String[] order4 = {"数据分析-主体信息汇总","数据分析-账户透视","数据分析-对手透视", "数据分析-共同关系网络"};
    @Autowired
    private GridFsTemplate gridFsTemplate;
    @Autowired
    private MongoTemplate mongoTemplate;
    @Autowired
    private GridFsOperations fsOperations;
    @Autowired
    @Lazy
    private FileService fileService;

    public FileService() {
    }

    public static String getCharsetName(File file) throws IOException {
        String charsetName = "UTF-8";
        // 获取 CodepageDetectorProxy 实例
        CodepageDetectorProxy detector = CodepageDetectorProxy.getInstance();
        // 添加解析器，会使用到添加的后 2 个 ext 里的 jar 包
        detector.add(new ParsingDetector(false));
        detector.add(JChardetFacade.getInstance());
        detector.add(ASCIIDetector.getInstance());
        detector.add(UnicodeDetector.getInstance());
        Charset charset = detector.detectCodepage(file.toURI().toURL());
        if (charset != null) {
            charsetName = charset.name();
        }
        return charsetName;
    }

    public static void main(String[] args) throws IOException {
        File file =
                new File(
                        "C:\\Users\\Administrator\\Desktop\\楼\\支付宝五联单数据格式\\20120303组织卖淫案_OR222713D2400411_注册信息_20210913100124.csv");
        String charsetName = getCharsetName(file);
        CSVParser csvParser =
                CSVFormat.EXCEL.parse(new InputStreamReader(new FileInputStream(file), charsetName));
        for (CSVRecord strings : csvParser) {
            System.out.println(strings);
        }

        file = new File("C:\\Users\\Administrator\\Desktop\\楼\\微信支付账单(2).csv");
        charsetName = getCharsetName(file);
        csvParser =
                CSVFormat.EXCEL.parse(new InputStreamReader(new FileInputStream(file), charsetName));
        for (CSVRecord strings : csvParser) {
            System.out.println(strings);
        }
    }

    public Mapped storeUploaded(
            InputStream inputStream, String encoding, String filename, String contentType, String modelId) {
        ObjectId objectId;
        objectId = this.gridFsTemplate.store(inputStream, filename, contentType);
        Query query =
                new Query(
                        Criteria.where("modelId")
                                .is(modelId)
                                .and("filename")
                                .is(filename));
        List<LogicalFile> existsFileList = this.mongoTemplate.find(query, LogicalFile.class);
        if (existsFileList != null && existsFileList.size() > 0) {
            return Mapped.OK();
        }
        LogicalFile logicalFile = new LogicalFile();
        logicalFile.setFilename(filename);
        logicalFile.setUploadDate(DateUtil.now());
        logicalFile.setModelId(modelId);
        logicalFile.setGridId(objectId);
        logicalFile.setGenerated(Boolean.FALSE);
        logicalFile.setStatus("LOADING");
        logicalFile = this.mongoTemplate.insert(logicalFile);
        this.readFile(logicalFile, encoding);
        return Mapped.OK();
    }

    @Async
    public void readFile(LogicalFile uploaded, String encoding) {
        GridFSFile fsFile =
                this.fsOperations.findOne(new Query(Criteria.where("_id").is(uploaded.getGridId())));
        if (fsFile == null) {
            this.modifyStatus(uploaded.getId(), "ERROR");
        } else {
            GridFsResource resource = this.fsOperations.getResource(fsFile);
            String filename = resource.getFilename();
            if (StringUtils.isBlank(filename)) {
                this.modifyStatus(uploaded.getId(), "ERROR");
            } else {
                try {
                    InputStream inputStream = resource.getInputStream();
                    MongoExcelService mongoExcelService;
                    if (filename.endsWith(".xlsx")) {
                        mongoExcelService = new MongoExcelService(this.mongoTemplate, uploaded.getId());
                        ExcelXlsxReader reader = new ExcelXlsxReader(mongoExcelService);
                        reader.processStream(inputStream);
                        mongoExcelService.insertRest();
                    } else if (filename.endsWith(".xls")) {
                        mongoExcelService = new MongoExcelService(this.mongoTemplate, uploaded.getId());
                        ExcelXlsReader reader = new ExcelXlsReader(mongoExcelService);
                        reader.processStream(inputStream);
                        mongoExcelService.insertRest();
                    } else if (filename.endsWith(".csv")) {
                        mongoExcelService = new MongoExcelService(this.mongoTemplate, uploaded.getId());
                        log.info("encoding2:{}", encoding);
                        CSVParser csvParser = null;
                        if (encoding.equalsIgnoreCase("UTF-8")) {
                            csvParser = CSVFormat.EXCEL.parse(new InputStreamReader(inputStream, "utf-8"));
                        } else {
                            csvParser = CSVFormat.EXCEL.parse(new InputStreamReader(inputStream));
                        }
                        int curRow = 0;
                        int curRowWxFile = 0;
                        boolean wxFile = false;
                        String nickname = ""; // 微信手机账单文件  添加一列 微信昵称  需要单独处理
                        for (CSVRecord record : csvParser) {
                            log.info("record,{}", record);
                            List<String> strings = record.toList();
                            List<String> strings2 = new ArrayList<>();
                            strings2.addAll(strings);
                            /*************      此处内容一下均为处理微信账单表头             ****************/
                            String join = String.join("", strings);
                            join = PublicStatic.removeAllSpecial(join);
                            if (join.equals("微信支付账单明细")) {
                                wxFile = true;
                            }
                            if (wxFile && curRowWxFile < 16) {
                                if (curRowWxFile == 1) {
                                    String temp = strings.get(0);
                                    if (StringUtils.isNotBlank(temp)) {
                                        nickname = temp.replace("微信昵称：[", "").replace("]", "");
                                    }
                                }
                                curRowWxFile++;
                                continue;
                            }
                            if (wxFile && curRowWxFile > 16) {
                                strings2.add(0, nickname);
                                curRowWxFile++;
                            }
                            if (wxFile && curRowWxFile == 16) {
                                strings2.add(0, "微信昵称");
                                curRowWxFile++;
                            }

                            /*************      此处内容以上均为处理微信账单表头             ****************/
                            mongoExcelService.getRows(
                                    0,
                                    curRow,
                                    strings2.stream()
                                            .map(PublicStatic::removeAllSpecial)
                                            .collect(Collectors.toList()));
                            curRow++;
                        }
                        mongoExcelService.insertRest();
                    } else if (filename.endsWith(".txt")) {
                        mongoExcelService = new MongoExcelService(this.mongoTemplate, uploaded.getId());
                        List<String> lines = new ArrayList<>();
                        IoUtil.readLines(new InputStreamReader(inputStream, "utf-8"), lines);
                        int curRow = 0;
                        for (String string : lines) {
                            if (string.equals("注销信息")) {
                                break;
                            }
                            List<String> values = Arrays.asList(string.split("\t"));
                            if (values.size() <= 1) {
                                continue;
                            }
                            mongoExcelService.getRows(
                                    0,
                                    curRow,
                                    values.stream().map(PublicStatic::removeAllSpecial).collect(Collectors.toList()));
                            curRow++;
                        }
                        mongoExcelService.insertRest();
                    }
                } catch (Exception var11) {
                    this.modifyStatus(uploaded.getId(), "ERROR");
                    var11.printStackTrace();
                }

                this.modifyStatus(uploaded.getId(), "FINISHED");
            }
        }
    }

    private void modifyStatus(String id, String status) {
        Query query = new Query();
        query.addCriteria(Criteria.where("_id").is(id));
        Update update = new Update();
        update.set("status", status);
        this.mongoTemplate.updateFirst(query, update, "logical_file");
    }

    public Mapped listGroupUploaded(FileListInput fileListInput) {
        String modelId = fileListInput.getModelId();
        if (modelId == null) {
            return Mapped.ERROR("没有选择任何分组");
        } else {
            Query query =
                    new Query(Criteria.where("modelId").is(modelId).and("generated").is(Boolean.TRUE));
            query.with(Sort.by(Order.desc("uploadDate")));
            List<LogicalFile> resultList = this.mongoTemplate.find(query, LogicalFile.class);
            // 将文件 按照  反诈  财付通 五联单  进行分类
            List<GroupLogincalFile> groupLogincalFiles = new ArrayList<>();
            groupLogicalFile("反诈治安数据分析", new String[]{"反诈", "治安","经侦"}, resultList, groupLogincalFiles);
            groupLogicalFile("支付宝数据分析", new String[]{"五联单"}, resultList, groupLogincalFiles);
            groupLogicalFile("财付通数据分析", new String[]{"财付通"}, resultList, groupLogincalFiles);
            groupLogicalFile("数据综合分析", new String[]{"数据分析"}, resultList, groupLogincalFiles);
            Mapped res = Mapped.OK();
            res.put("code", 200);
            res.put("result", groupLogincalFiles);
            return res;
        }
    }

    public void groupLogicalFile(
            String groupName,
            String[] group,
            List<LogicalFile> logicalFileList,
            List<GroupLogincalFile> groupLogincalFiles) {
        List<LogicalFile> children = new ArrayList<>();
        for (String s : group) {
            children.addAll(
                    logicalFileList.stream()
                            .filter(t -> t.getFilename().startsWith(s))
                            .collect(Collectors.toList()));
        }
        if (children != null && children.size() > 0) {
            GroupLogincalFile groupLogincalFile = new GroupLogincalFile();
            groupLogincalFile.setGroupName(groupName);
            groupLogincalFile.setId(IdUtil.randomUUID());
            groupLogincalFile.setChildren(orderGroupLogicalFile(groupName, children));
            groupLogincalFiles.add(groupLogincalFile);
        }
    }

    public List<LogicalFile> orderGroupLogicalFile(String groupName, List<LogicalFile> list) {

        if (groupName.equals("反诈治安数据分析")) {
            return orderGroupLogicalFileChild(order1, list);
        }
        if (groupName.equals("支付宝数据分析")) {
            return orderGroupLogicalFileChild(order2, list);
        }
        if (groupName.equals("财付通数据分析")) {
            return orderGroupLogicalFileChild(order3, list);
        }
        if (groupName.equals("数据综合分析")) {
            return orderGroupLogicalFileChild(order4, list);
        }
        return null;
    }

    public List<LogicalFile> orderGroupLogicalFileChild(String[] orders, List<LogicalFile> list) {
        List<LogicalFile> child = new ArrayList<>();
        for (String s : orders) {

            for (LogicalFile logicalFile : list) {
                if (logicalFile.getFilename().equals(s)) {
                    if("数据分析-共同关系网络".equals(logicalFile.getFilename())){
                        logicalFile.setFileType("map");
                    }
                    child.add(logicalFile);
                    break;
                }
            }
        }
        return child;
    }

    public Mapped listUploaded(FileListInput fileListInput) {
        String modelId = fileListInput.getModelId();
        if (modelId == null) {
            return Mapped.ERROR("没有选择任何分组");
        } else {
            String generated = fileListInput.getGenerated();
            Future<Long> longFuture = this.fileService.countTotalByModelId(modelId, generated);
            Future<String> modelNameFuture = this.fileService.retrieveModelname(modelId);
            int page = fileListInput.getPage();
            int pageSize = fileListInput.getPageSize();
            int skip = (page - 1) * pageSize;
            Criteria criteria = Criteria.where("modelId").is(modelId);
            if ("true".equals(generated)) {
                criteria.and("generated").is(Boolean.TRUE);
            } else if ("false".equals(generated)) {
                criteria.and("generated").is(Boolean.FALSE);
            }
            Query query = new Query(criteria);
            query.with(Sort.by(Order.desc("uploadDate")));
            List<LogicalFile> resultList =
                    this.mongoTemplate.find(query.skip(skip).limit(pageSize), LogicalFile.class);
            Long total;
            try {
                total = longFuture.get();
            } catch (ExecutionException | InterruptedException var17) {
                return Mapped.ERROR("查询出错");
            }
            String modelName;
            try {
                modelName = modelNameFuture.get();
            } catch (ExecutionException | InterruptedException var17) {
                return Mapped.ERROR("查询出错");
            }
            int i = skip;
            for (LogicalFile file : resultList) {
                ++i;
                file.setCount(i);
                file.setModelName(modelName);
            }
            Mapped res = Mapped.OK();
            res.put("code", 200);
            res.put("result", resultList);
            res.put("total", total);
            res.put("page", page);
            res.put("pageSize", pageSize);
            return res;
        }
    }

    public Mapped deleteById(IdInput idInput) {
        String id = idInput.getId();
        Query query = new Query(Criteria.where("id").is(id));
        LogicalFile file = this.mongoTemplate.findOne(query, LogicalFile.class);
        if (file != null && file.getGridId() != null) {
            String physicalId = file.getGridId().toHexString();
            this.fileService.removePhysicalFile(physicalId);
        }

        this.fileService.removeDataByFileId(id);
        this.fileService.removeFieldsByFileId(id);
        DeleteResult result = this.mongoTemplate.remove(query, LogicalFile.class);
        return result.getDeletedCount() > 0L ? Mapped.OK() : Mapped.ERROR("删除失败");
    }

    @Async
    Future<Long> countTotalByModelId(String modelId, String generated) {
        Criteria criteria = Criteria.where("modelId").is(modelId);
        if ("true".equals(generated)) {
            criteria.and("generated").is(Boolean.TRUE);
        } else if ("false".equals(generated)) {
            criteria.and("generated").is(Boolean.FALSE);
        }

        Long total = this.mongoTemplate.count(new Query(criteria), LogicalFile.class);
        return new AsyncResult(total);
    }

    @Async
    public void removePhysicalFile(String id) {
        this.fsOperations.delete(new Query(Criteria.where("_id").is(id)));
    }

    @Async
    public void removeLogicalFile(String fileId) {
        this.mongoTemplate.remove(new Query(Criteria.where("id").is(fileId)), LogicalFile.class);
    }

    @Async
    public void removeDataByFileId(String fileId) {
        this.mongoTemplate.remove(new Query(Criteria.where("fileId").is(fileId)), DataMap.class);
    }

    @Async
    public void removeFieldsByFileId(String fileId) {
        this.mongoTemplate.remove(new Query(Criteria.where("fileId").is(fileId)), Fields.class);
    }

    @Async
    Future<String> retrieveModelname(String modelId) {
        Query query = new Query(Criteria.where("id").is(modelId));
        query.fields().include("modelName");
        Model model = this.mongoTemplate.findOne(query, Model.class, "model");
        return new AsyncResult(model != null ? model.getModelName() : "");
    }
}