createmodel-java/src/main/java/ieven/server/webapp/domain/file/FileService.java

//
// Source code recreated from a .class file by IntelliJ IDEA
// (powered by FernFlower decompiler)
//

package ieven.server.webapp.domain.file;

import cn.hutool.core.date.DateUtil;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.io.IoUtil;
import cn.hutool.core.util.IdUtil;
import cn.hutool.poi.excel.ExcelReader;
import cn.hutool.poi.excel.ExcelUtil;
import com.mongodb.client.gridfs.model.GridFSFile;
import com.mongodb.client.result.DeleteResult;
import com.opencsv.CSVReader;
import ieven.server.webapp.domain.IdInput;
import ieven.server.webapp.domain.data.DataMap;
import ieven.server.webapp.domain.data.Fields;
import ieven.server.webapp.domain.model.Model;
import ieven.server.webapp.infrastructure.wrapper.Mapped;
import ieven.server.webapp.service.MongoExcelService;
import ieven.server.webapp.util.EncodeDetector;
import ieven.server.webapp.util.FileUtils;
import ieven.server.webapp.util.excel.ExcelXlsReader;
import ieven.server.webapp.util.excel.ExcelXlsxReader;
import ieven.server.webapp.util.excel.PublicStatic;
import info.monitorenter.cpdetector.io.*;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.apache.commons.lang3.StringUtils;
import org.apache.poi.ss.usermodel.Sheet;
import org.bson.types.ObjectId;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.data.domain.Sort;
import org.springframework.data.domain.Sort.Order;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update;
import org.springframework.data.mongodb.gridfs.GridFsOperations;
import org.springframework.data.mongodb.gridfs.GridFsResource;
import org.springframework.data.mongodb.gridfs.GridFsTemplate;
import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.annotation.AsyncResult;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;

import java.io.*;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.*;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

@Service
@EnableAsync
@Slf4j
public class FileService {
    @Value("${spring.data.fileExportPath}")
    private String fileUploadPath;

    String[] order1 = {"反诈数据-订单明细", "反诈数据-订单明细2", "治安数据-订单明细", "经侦数据-订单明细", "网黑数据-订单明细", "反诈治安-交易明细", "反诈治安-账户透视", "反诈治安-对手透视"};
    String[] order2 = {
            "五联单-注册信息",
            "经侦云-注册信息",
            "五联单-登录日志",
            "五联单-交易记录",
            "五联单-账户明细",
            "经侦云-账户明细",
            "五联单-转账明细",
            "五联单-转账分析",
            "五联单-账户透视",
            "五联单-对手透视",
            "五联单-IP地址分析",
            "五联单-收货地址分析"
    };
    String[] order3 = {"财付通-注册信息", "财付通-订单明细1", "财付通-订单明细2","财付通-商户订单", "财付通-交易明细", "财付通-账户透视", "财付通-对手透视", "财付通-手机明细"};

    String[] order4 = {"数据分析-主体信息汇总", "数据分析-账户透视", "数据分析-对手透视", "数据分析-共同关系网络"};
    @Autowired
    private GridFsTemplate gridFsTemplate;
    @Autowired
    private MongoTemplate mongoTemplate;
    @Autowired
    private GridFsOperations fsOperations;
    @Autowired
    @Lazy
    private FileService fileService;

    public FileService() {
    }

    public static String getCharsetName(File file) throws IOException {
        String charsetName = "UTF-8";
        // 获取 CodepageDetectorProxy 实例
        CodepageDetectorProxy detector = CodepageDetectorProxy.getInstance();
        // 添加解析器，会使用到添加的后 2 个 ext 里的 jar 包
        detector.add(new ParsingDetector(false));
        detector.add(JChardetFacade.getInstance());
        detector.add(ASCIIDetector.getInstance());
        detector.add(UnicodeDetector.getInstance());
        Charset charset = detector.detectCodepage(file.toURI().toURL());
        if (charset != null) {
            charsetName = charset.name();
        }
        return charsetName;
    }

    public static void main(String[] args) throws IOException {
        File file =
                new File(
                        "C:\\Users\\Administrator\\Desktop\\楼\\支付宝五联单数据格式\\20120303组织卖淫案_OR222713D2400411_注册信息_20210913100124.csv");
        String charsetName = getCharsetName(file);
        CSVParser csvParser =
                CSVFormat.EXCEL.parse(new InputStreamReader(new FileInputStream(file), charsetName));
        for (CSVRecord strings : csvParser) {
            System.out.println(strings);
        }

        file = new File("C:\\Users\\Administrator\\Desktop\\楼\\微信支付账单(2).csv");
        charsetName = getCharsetName(file);
        csvParser =
                CSVFormat.EXCEL.parse(new InputStreamReader(new FileInputStream(file), charsetName));
        for (CSVRecord strings : csvParser) {
            System.out.println(strings);
        }
    }

    public Mapped storeUploaded(
            InputStream inputStream, String encoding, String filename, String contentType, String modelId) {
        ObjectId objectId;
        objectId = this.gridFsTemplate.store(inputStream, filename, contentType);
        Query query =
                new Query(
                        Criteria.where("modelId")
                                .is(modelId)
                                .and("filename")
                                .is(filename));
        List<LogicalFile> existsFileList = this.mongoTemplate.find(query, LogicalFile.class);
        if (existsFileList != null && existsFileList.size() > 0) {
            return Mapped.OK();
        }
        LogicalFile logicalFile = new LogicalFile();
        logicalFile.setFilename(filename);
        logicalFile.setUploadDate(DateUtil.now());
        logicalFile.setModelId(modelId);
        logicalFile.setGridId(objectId);
        logicalFile.setGenerated(Boolean.FALSE);
        logicalFile.setStatus("LOADING");
        logicalFile = this.mongoTemplate.insert(logicalFile);
        this.readFile(logicalFile, encoding);
        return Mapped.OK();
    }

    @Async
    public void readFile(LogicalFile uploaded, String encoding) {
        GridFSFile fsFile =
                this.fsOperations.findOne(new Query(Criteria.where("_id").is(uploaded.getGridId())));
        if (fsFile == null) {
            this.modifyStatus(uploaded.getId(), "ERROR");
        } else {
            GridFsResource resource = this.fsOperations.getResource(fsFile);
            String filename = resource.getFilename();
            if (StringUtils.isBlank(filename)) {
                this.modifyStatus(uploaded.getId(), "ERROR");
            } else {
                try {
                    InputStream inputStream = resource.getInputStream();
                    MongoExcelService mongoExcelService;
                    if (filename.endsWith(".xlsx")) {
                        mongoExcelService = new MongoExcelService(this.mongoTemplate, uploaded.getId());
                        ExcelXlsxReader reader = new ExcelXlsxReader(mongoExcelService);
                        reader.processStream(inputStream);
                        mongoExcelService.insertRest();
                    } else if (filename.endsWith(".xls")) {
                        mongoExcelService = new MongoExcelService(this.mongoTemplate, uploaded.getId());
                        ExcelXlsReader reader = new ExcelXlsReader(mongoExcelService);
                        reader.processStream(inputStream);
                        mongoExcelService.insertRest();
                    } else if (filename.endsWith(".csv")) {
                        mongoExcelService = new MongoExcelService(this.mongoTemplate, uploaded.getId());
                        log.info("encoding2:{}", encoding);
                        /*CSVParser csvParser = null;
                        if (encoding.equalsIgnoreCase("UTF-8")) {
                            csvParser = CSVFormat.EXCEL.parse(new InputStreamReader(inputStream, "utf-8"));
                        } else {
                            csvParser = CSVFormat.EXCEL.parse(new InputStreamReader(inputStream));
                        }*/
                        int curRow = 0;
                        int curRowWxFile = 0;
                        boolean wxFile = false;
                        String nickname = ""; // 微信手机账单文件  添加一列 微信昵称  需要单独处理
                        try (CSVReader reader = new CSVReader(new InputStreamReader(inputStream))) {
                            List<String[]> lines = reader.readAll();
                            for (String[] line : lines) {
                                log.info("record,{}", line);
                                List<String> strings = Arrays.asList(line);
                                List<String> strings2 = new ArrayList<>();
                                strings2.addAll(strings);
                                /*************      此处内容一下均为处理微信账单表头             ****************/
                                String join = String.join("", strings);
                                join = PublicStatic.removeAllSpecial(join);
                                if (join.equals("微信支付账单明细")) {
                                    wxFile = true;
                                }
                                if (wxFile && curRowWxFile < 16) {
                                    if (curRowWxFile == 1) {
                                        String temp = strings.get(0);
                                        if (StringUtils.isNotBlank(temp)) {
                                            nickname = temp.replace("微信昵称：[", "").replace("]", "");
                                        }
                                    }
                                    curRowWxFile++;
                                    continue;
                                }
                                if (wxFile && curRowWxFile > 16) {
                                    strings2.add(0, nickname);
                                    curRowWxFile++;
                                }
                                if (wxFile && curRowWxFile == 16) {
                                    strings2.add(0, "微信昵称");
                                    curRowWxFile++;
                                }

                                /*************      此处内容以上均为处理微信账单表头             ****************/
                                mongoExcelService.getRows(
                                        0,
                                        curRow,
                                        strings2.stream()
                                                .map(PublicStatic::removeAllSpecial)
                                                .collect(Collectors.toList())
                                );
                                curRow++;
                            }

                        } catch (Exception e) {
                            e.printStackTrace();
                        }
                        mongoExcelService.insertRest();
                    } else if (filename.endsWith(".txt")) {
                        mongoExcelService = new MongoExcelService(this.mongoTemplate, uploaded.getId());
                        List<String> lines = new ArrayList<>();
                        IoUtil.readLines(new InputStreamReader(inputStream, "utf-8"), lines);
                        int curRow = 0;
                        for (String string : lines) {
                            if (string.equals("注销信息")) {
                                break;
                            }
                            List<String> values = Arrays.asList(string.split("\t"));
                            if (values.size() <= 1) {
                                continue;
                            }
                            mongoExcelService.getRows(
                                    0,
                                    curRow,
                                    values.stream().map(PublicStatic::removeAllSpecial).collect(Collectors.toList())
                            );
                            curRow++;
                        }
                        mongoExcelService.insertRest();
                    }
                } catch (Exception var11) {
                    this.modifyStatus(uploaded.getId(), "ERROR");
                    var11.printStackTrace();
                }

                this.modifyStatus(uploaded.getId(), "FINISHED");
            }
        }
    }

    private void modifyStatus(String id, String status) {
        Query query = new Query();
        query.addCriteria(Criteria.where("_id").is(id));
        Update update = new Update();
        update.set("status", status);
        this.mongoTemplate.updateFirst(query, update, "logical_file");
    }

    public Mapped listGroupUploaded(FileListInput fileListInput) {
        String modelId = fileListInput.getModelId();
        if (modelId == null) {
            return Mapped.ERROR("没有选择任何分组");
        } else {
            Query query =
                    new Query(Criteria.where("modelId").is(modelId).and("generated").is(Boolean.TRUE));
            query.with(Sort.by(Order.desc("uploadDate")));
            List<LogicalFile> resultList = this.mongoTemplate.find(query, LogicalFile.class);
            // 将文件 按照  反诈  财付通 五联单  进行分类
            List<GroupLogincalFile> groupLogincalFiles = new ArrayList<>();
            groupLogicalFile("反诈治安数据分析", new String[]{"反诈", "治安", "经侦数据", "网黑"}, resultList, groupLogincalFiles);
            groupLogicalFile("支付宝数据分析", new String[]{"五联单", "经侦云"}, resultList, groupLogincalFiles);
            groupLogicalFile("财付通数据分析", new String[]{"财付通"}, resultList, groupLogincalFiles);
            groupLogicalFile("数据综合分析", new String[]{"数据分析"}, resultList, groupLogincalFiles);
            Mapped res = Mapped.OK();
            res.put("code", 200);
            res.put("result", groupLogincalFiles);
            return res;
        }
    }

    public void groupLogicalFile(
            String groupName,
            String[] group,
            List<LogicalFile> logicalFileList,
            List<GroupLogincalFile> groupLogincalFiles) {
        List<LogicalFile> children = new ArrayList<>();
        for (String s : group) {
            children.addAll(
                    logicalFileList.stream()
                            .filter(t -> t.getFilename().startsWith(s))
                            .collect(Collectors.toList()));
        }
        if (children != null && children.size() > 0) {
            GroupLogincalFile groupLogincalFile = new GroupLogincalFile();
            groupLogincalFile.setGroupName(groupName);
            groupLogincalFile.setId(IdUtil.randomUUID());
            groupLogincalFile.setChildren(orderGroupLogicalFile(groupName, children));
            groupLogincalFiles.add(groupLogincalFile);
        }
    }

    public List<LogicalFile> orderGroupLogicalFile(String groupName, List<LogicalFile> list) {

        if (groupName.equals("反诈治安数据分析")) {
            return orderGroupLogicalFileChild(order1, list);
        }
        if (groupName.equals("支付宝数据分析")) {
            return orderGroupLogicalFileChild(order2, list);
        }
        if (groupName.equals("财付通数据分析")) {
            return orderGroupLogicalFileChild(order3, list);
        }
        if (groupName.equals("数据综合分析")) {
            return orderGroupLogicalFileChild(order4, list);
        }
        return null;
    }

    public List<LogicalFile> orderGroupLogicalFileChild(String[] orders, List<LogicalFile> list) {
        List<LogicalFile> child = new ArrayList<>();
        for (String s : orders) {

            for (LogicalFile logicalFile : list) {
                if (logicalFile.getFilename().equals(s)) {
                    if ("数据分析-共同关系网络".equals(logicalFile.getFilename())) {
                        logicalFile.setFileType("map");
                    }
                    child.add(logicalFile);
                    break;
                }
            }
        }
        return child;
    }

    public Mapped listUploaded(FileListInput fileListInput) {
        String modelId = fileListInput.getModelId();
        if (modelId == null) {
            return Mapped.ERROR("没有选择任何分组");
        } else {
            String generated = fileListInput.getGenerated();
            String fileName = fileListInput.getFileName();
            Future<Long> longFuture = this.fileService.countTotalByModelId(modelId, generated);
            Future<String> modelNameFuture = this.fileService.retrieveModelname(modelId);
            int page = fileListInput.getPage();
            int pageSize = fileListInput.getPageSize();
            int skip = (page - 1) * pageSize;
            Criteria criteria = Criteria.where("modelId").is(modelId);
            if ("true".equals(generated)) {
                criteria.and("generated").is(Boolean.TRUE);
            } else if ("false".equals(generated)) {
                criteria.and("generated").is(Boolean.FALSE);
            }

            if (fileName != null && !fileName.isEmpty()) {
                // 使用正则表达式进行模糊匹配，i 表示不区分大小写
                Pattern pattern = Pattern.compile(".*" + Pattern.quote(fileName) + ".*", Pattern.CASE_INSENSITIVE);
                criteria.and("filename").regex(pattern);
            }

            // 创建查询对象并执行查询
            Query query = new Query(criteria)
                    .with(Sort.by(Sort.Direction.DESC, "uploadDate")) // 保留排序逻辑
                    .skip(skip)
                    .limit(pageSize);
            List<LogicalFile> resultList =
                    this.mongoTemplate.find(query, LogicalFile.class);
            Long total;
            try {
                total = longFuture.get();
            } catch (ExecutionException | InterruptedException var17) {
                return Mapped.ERROR("查询出错");
            }
            String modelName;
            try {
                modelName = modelNameFuture.get();
            } catch (ExecutionException | InterruptedException var17) {
                return Mapped.ERROR("查询出错");
            }
            int i = skip;
            for (LogicalFile file : resultList) {
                ++i;
                file.setCount(i);
                file.setModelName(modelName);
            }
            Mapped res = Mapped.OK();
            res.put("code", 200);
            res.put("result", resultList);
            res.put("total", total);
            res.put("page", page);
            res.put("pageSize", pageSize);
            return res;
        }
    }

    public Mapped deleteById(IdInput idInput) {
        String id = idInput.getId();
        Query query = new Query(Criteria.where("id").is(id));
        LogicalFile file = this.mongoTemplate.findOne(query, LogicalFile.class);
        if (file != null && file.getGridId() != null) {
            String physicalId = file.getGridId().toHexString();
            this.fileService.removePhysicalFile(physicalId);
        }

        this.fileService.removeDataByFileId(id);
        this.fileService.removeFieldsByFileId(id);
        DeleteResult result = this.mongoTemplate.remove(query, LogicalFile.class);
        return result.getDeletedCount() > 0L ? Mapped.OK() : Mapped.ERROR("删除失败");
    }

    @Async
    Future<Long> countTotalByModelId(String modelId, String generated) {
        Criteria criteria = Criteria.where("modelId").is(modelId);
        if ("true".equals(generated)) {
            criteria.and("generated").is(Boolean.TRUE);
        } else if ("false".equals(generated)) {
            criteria.and("generated").is(Boolean.FALSE);
        }

        Long total = this.mongoTemplate.count(new Query(criteria), LogicalFile.class);
        return new AsyncResult(total);
    }

    @Async
    public void removePhysicalFile(String id) {
        this.fsOperations.delete(new Query(Criteria.where("_id").is(id)));
    }

    @Async
    public void removeLogicalFile(String fileId) {
        this.mongoTemplate.remove(new Query(Criteria.where("id").is(fileId)), LogicalFile.class);
    }

    @Async
    public void removeDataByFileId(String fileId) {
        this.mongoTemplate.remove(new Query(Criteria.where("fileId").is(fileId)), DataMap.class);
    }

    @Async
    public void removeFieldsByFileId(String fileId) {
        this.mongoTemplate.remove(new Query(Criteria.where("fileId").is(fileId)), Fields.class);
    }

    @Async
    Future<String> retrieveModelname(String modelId) {
        Query query = new Query(Criteria.where("id").is(modelId));
        query.fields().include("modelName");
        Model model = this.mongoTemplate.findOne(query, Model.class, "model");
        return new AsyncResult(model != null ? model.getModelName() : "");
    }

    public Mapped uploadFilesAndSave(List<MultipartFile> multipartFiles, String modelId) {
        List<String> failNameList = new ArrayList<>();
        for (int i = 0; i < multipartFiles.size(); i++) {
            MultipartFile multipartFile = multipartFiles.get(i);
            String originName = multipartFile.getOriginalFilename();
            String suffix = originName.substring(originName.lastIndexOf(".") + 1);
            String prefix = originName.substring(0, originName.lastIndexOf("."));
            String savedName = prefix + "(" + (i + 1) + ")." + suffix;
            String filepath = fileUploadPath + File.separator + modelId + File.separator + savedName;
            FileUtils.uploadFile(multipartFile, filepath);
            MultipartFile multipartFile1 = FileUtils.getMultipartFile(new File(filepath));
            try {
                ExecutorService executor = Executors.newSingleThreadExecutor();
                Future<Mapped> future = executor.submit(new Callable<Mapped>() {
                    @Override
                    public Mapped call() throws Exception {
                        return uploadSingle(multipartFile1, modelId);
                    }
                });

                // 在这里，主线程会等待 future 的结果
                Mapped mapped = future.get(); // 这将会阻塞，直到Callable任务完成
                if (200 != Integer.valueOf(String.valueOf(mapped.get("code")))) {
                    failNameList.add(savedName + ":" + String.valueOf(mapped.get("message")));
                }
                //删除文件
                FileUtils.deleteFile(filepath);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
        Mapped mapped = Mapped.OK();
        mapped.put("failNameList", failNameList);
        return mapped;
    }

    public Mapped uploadSingle(MultipartFile file, String modelId)
            throws Exception {
        String filename = file.getOriginalFilename();
        String extName = FileUtil.extName(filename);
        String contentType = file.getContentType();
        ByteArrayOutputStream baos = FileUtils.cloneInputStream(file.getInputStream());
        if (extName.equalsIgnoreCase("xls") || extName.equalsIgnoreCase("xlsx")) {
            ExcelReader reader = ExcelUtil.getReader(new ByteArrayInputStream(baos.toByteArray()));
            List<Sheet> sheets = reader.getSheets();
            List<String> headers = new ArrayList<>();
            for (Sheet sheet : sheets) {
                reader.setSheet(sheet);
                List<Object> objects = reader.readRow(0);
                if (objects.size() > 1) { // 至少大于1
                    StringBuilder stringBuilder = new StringBuilder();
                    for (Object object : objects) {
                        stringBuilder.append(object);
                    }
                    headers.add(stringBuilder.toString());
                }
            }
            if (headers.size() > 1) {
                // 判断sheet第一行内容是否一致
                StringBuilder errorHeaders = new StringBuilder();
                String baseHeader = headers.get(0);
                for (int i = 0; i < headers.size(); i++) {
                    //
                    if (i == 0) {
                        continue;
                    }
                    if (!headers.get(i).equals(baseHeader)) {
                        errorHeaders.append("sheet").append(i).append("标题列与第一个sheet不一致!");
                    }
                }
                if (errorHeaders.length() > 0) {
                    return Mapped.ERROR(errorHeaders.toString());
                }
            }
            reader.close();
        }
        String encoding = EncodeDetector.getEncoding(baos.toByteArray());
        baos.close();
        return this.fileService.storeUploaded(
                new ByteArrayInputStream(baos.toByteArray()), encoding, filename, contentType, modelId);
    }

}