// // Source code recreated from a .class file by IntelliJ IDEA // (powered by FernFlower decompiler) // package ieven.server.webapp.domain.file; import cn.hutool.core.date.DateUtil; import cn.hutool.core.io.IoUtil; import cn.hutool.core.util.IdUtil; import com.mongodb.client.gridfs.model.GridFSFile; import com.mongodb.client.result.DeleteResult; import ieven.server.webapp.domain.IdInput; import ieven.server.webapp.domain.data.DataMap; import ieven.server.webapp.domain.data.Fields; import ieven.server.webapp.domain.model.Model; import ieven.server.webapp.infrastructure.wrapper.Mapped; import ieven.server.webapp.service.MongoExcelService; import ieven.server.webapp.util.excel.ExcelXlsReader; import ieven.server.webapp.util.excel.ExcelXlsxReader; import ieven.server.webapp.util.excel.PublicStatic; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; import org.apache.commons.lang3.StringUtils; import org.apache.poi.openxml4j.exceptions.OpenXML4JException; import org.bson.types.ObjectId; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.data.domain.Sort; import org.springframework.data.domain.Sort.Order; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Update; import org.springframework.data.mongodb.gridfs.GridFsOperations; import org.springframework.data.mongodb.gridfs.GridFsResource; import org.springframework.data.mongodb.gridfs.GridFsTemplate; import org.springframework.scheduling.annotation.Async; import org.springframework.scheduling.annotation.AsyncResult; import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.stereotype.Service; import org.xml.sax.SAXException; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.stream.Collectors; @Service @EnableAsync public class FileService { @Autowired private GridFsTemplate gridFsTemplate; @Autowired private MongoTemplate mongoTemplate; @Autowired private GridFsOperations fsOperations; @Autowired @Lazy private FileService fileService; public FileService() {} public Mapped storeUploaded( InputStream inputStream, String filename, String contentType, String modelId) { ObjectId objectId; objectId = this.gridFsTemplate.store(inputStream, filename, contentType); LogicalFile logicalFile = new LogicalFile(); logicalFile.setFilename(filename); logicalFile.setUploadDate(DateUtil.now()); logicalFile.setModelId(modelId); logicalFile.setGridId(objectId); logicalFile.setGenerated(Boolean.FALSE); logicalFile.setStatus("LOADING"); logicalFile = this.mongoTemplate.insert(logicalFile); this.readFile(logicalFile); return Mapped.OK(); } @Async public void readFile(LogicalFile uploaded) { GridFSFile fsFile = this.fsOperations.findOne(new Query(Criteria.where("_id").is(uploaded.getGridId()))); if (fsFile == null) { this.modifyStatus(uploaded.getId(), "ERROR"); } else { GridFsResource resource = this.fsOperations.getResource(fsFile); String filename = resource.getFilename(); if (StringUtils.isBlank(filename)) { this.modifyStatus(uploaded.getId(), "ERROR"); } else { try { InputStream inputStream = resource.getInputStream(); MongoExcelService mongoExcelService; if (filename.endsWith(".xlsx")) { mongoExcelService = new MongoExcelService(this.mongoTemplate, uploaded.getId()); ExcelXlsxReader reader = new ExcelXlsxReader(mongoExcelService); reader.processStream(inputStream); mongoExcelService.insertRest(); } else if (filename.endsWith(".xls")) { mongoExcelService = new MongoExcelService(this.mongoTemplate, uploaded.getId()); ExcelXlsReader reader = new ExcelXlsReader(mongoExcelService); reader.processStream(inputStream); mongoExcelService.insertRest(); } else if (filename.endsWith(".csv")) { mongoExcelService = new MongoExcelService(this.mongoTemplate, uploaded.getId()); CSVParser csvParser = CSVFormat.EXCEL.parse(new InputStreamReader(inputStream, "gbk")); int curRow = 0; int curRowWxFile = 0; boolean wxFile = false; String nickname = ""; // 微信手机账单文件 添加一列 微信昵称 需要单独处理 for (CSVRecord record : csvParser) { List strings = record.toList(); /************* 此处内容一下均为处理微信账单表头 ****************/ if (strings.size() == 1 && strings.get(0).equals("微信支付账单明细")) { wxFile = true; } if (wxFile && curRowWxFile < 16) { if (curRowWxFile == 1) { String temp = strings.get(0); if (StringUtils.isNotBlank(temp)) { nickname = temp.replace("微信昵称:[", "").replace("]", ""); } } curRowWxFile++; continue; } if (wxFile && curRowWxFile == 16) { strings.add(0, "微信昵称"); curRowWxFile++; } if (wxFile && curRowWxFile > 16) { strings.add(0, nickname); curRowWxFile++; } /************* 此处内容以上均为处理微信账单表头 ****************/ mongoExcelService.getRows( 0, curRow, strings.stream() .map(PublicStatic::removeAllSpecial) .collect(Collectors.toList())); curRow++; } mongoExcelService.insertRest(); } else if (filename.endsWith(".txt")) { mongoExcelService = new MongoExcelService(this.mongoTemplate, uploaded.getId()); List lines = new ArrayList<>(); IoUtil.readLines(new InputStreamReader(inputStream, "utf-8"), lines); int curRow = 0; for (String string : lines) { if (string.equals("注销信息")) { break; } List values = Arrays.asList(string.split("\t")); if (values.size() <= 1) { continue; } mongoExcelService.getRows( 0, curRow, values.stream().map(PublicStatic::removeAllSpecial).collect(Collectors.toList())); curRow++; } mongoExcelService.insertRest(); } } catch (OpenXML4JException | SAXException | IOException var11) { this.modifyStatus(uploaded.getId(), "ERROR"); var11.printStackTrace(); } this.modifyStatus(uploaded.getId(), "FINISHED"); } } } private void modifyStatus(String id, String status) { Query query = new Query(); query.addCriteria(Criteria.where("_id").is(id)); Update update = new Update(); update.set("status", status); this.mongoTemplate.updateFirst(query, update, "logical_file"); } public Mapped listGroupUploaded(FileListInput fileListInput) { String modelId = fileListInput.getModelId(); if (modelId == null) { return Mapped.ERROR("没有选择任何分组"); } else { Query query = new Query(Criteria.where("modelId").is(modelId).and("generated").is(Boolean.TRUE)); query.with(Sort.by(Order.desc("uploadDate"))); List resultList = this.mongoTemplate.find(query, LogicalFile.class); // 将文件 按照 反诈 财付通 五联单 进行分类 List groupLogincalFiles = new ArrayList<>(); groupLogicalFile("反诈治安数据分析", new String[] {"反诈", "治安"}, resultList, groupLogincalFiles); groupLogicalFile("支付宝数据分析", new String[] {"五联单"}, resultList, groupLogincalFiles); groupLogicalFile("财付通数据分析", new String[] {"财付通"}, resultList, groupLogincalFiles); Mapped res = Mapped.OK(); res.put("code", 200); res.put("result", groupLogincalFiles); return res; } } public void groupLogicalFile( String groupName, String[] group, List logicalFileList, List groupLogincalFiles) { List children = new ArrayList<>(); for (String s : group) { children.addAll( logicalFileList.stream() .filter(t -> t.getFilename().startsWith(s)) .collect(Collectors.toList())); } if (children != null && children.size() > 0) { GroupLogincalFile groupLogincalFile = new GroupLogincalFile(); groupLogincalFile.setGroupName(groupName); groupLogincalFile.setId(IdUtil.randomUUID()); groupLogincalFile.setChildren(orderGroupLogicalFile(groupName, children)); groupLogincalFiles.add(groupLogincalFile); } } String[] order1 = {"反诈数据-订单明细", "治安数据-订单明细", "反诈治安-交易明细", "反诈治安-账户透视", "反诈治安-对手透视"}; String[] order2 = { "五联单-注册信息", "五联单-登录日志", "五联单-交易记录", "五联单-账户明细", "五联单-转账明细", "五联单-转账分析", "五联单-账户透视", "五联单-对手透视", "五联单-IP地址分析", "五联单-收货地址分析" }; String[] order3 = {"财付通-开户信息", "财付通-订单明细", "财付通-交易明细", "财付通-账户透视", "财付通-对手透视", "财付通-手机明细"}; public List orderGroupLogicalFile(String groupName, List list) { if (groupName.equals("反诈治安数据分析")) { return orderGroupLogicalFileChild(order1, list); } if (groupName.equals("支付宝数据分析")) { return orderGroupLogicalFileChild(order2, list); } if (groupName.equals("财付通数据分析")) { return orderGroupLogicalFileChild(order3, list); } return null; } public List orderGroupLogicalFileChild(String[] orders, List list) { List child = new ArrayList<>(); for (String s : orders) { for (LogicalFile logicalFile : list) { if (logicalFile.getFilename().equals(s)) { child.add(logicalFile); break; } } } return child; } public Mapped listUploaded(FileListInput fileListInput) { String modelId = fileListInput.getModelId(); if (modelId == null) { return Mapped.ERROR("没有选择任何分组"); } else { String generated = fileListInput.getGenerated(); Future longFuture = this.fileService.countTotalByModelId(modelId, generated); Future modelNameFuture = this.fileService.retrieveModelname(modelId); int page = fileListInput.getPage(); int pageSize = fileListInput.getPageSize(); int skip = (page - 1) * pageSize; Criteria criteria = Criteria.where("modelId").is(modelId); if ("true".equals(generated)) { criteria.and("generated").is(Boolean.TRUE); } else if ("false".equals(generated)) { criteria.and("generated").is(Boolean.FALSE); } Query query = new Query(criteria); query.with(Sort.by(Order.desc("uploadDate"))); List resultList = this.mongoTemplate.find(query.skip(skip).limit(pageSize), LogicalFile.class); Long total; try { total = longFuture.get(); } catch (ExecutionException | InterruptedException var17) { return Mapped.ERROR("查询出错"); } String modelName; try { modelName = modelNameFuture.get(); } catch (ExecutionException | InterruptedException var17) { return Mapped.ERROR("查询出错"); } int i = skip; for (LogicalFile file : resultList) { ++i; file.setCount(i); file.setModelName(modelName); } Mapped res = Mapped.OK(); res.put("code", 200); res.put("result", resultList); res.put("total", total); res.put("page", page); res.put("pageSize", pageSize); return res; } } public Mapped deleteById(IdInput idInput) { String id = idInput.getId(); Query query = new Query(Criteria.where("id").is(id)); LogicalFile file = this.mongoTemplate.findOne(query, LogicalFile.class); if (file != null && file.getGridId() != null) { String physicalId = file.getGridId().toHexString(); this.fileService.removePhysicalFile(physicalId); } this.fileService.removeDataByFileId(id); this.fileService.removeFieldsByFileId(id); DeleteResult result = this.mongoTemplate.remove(query, LogicalFile.class); return result.getDeletedCount() > 0L ? Mapped.OK() : Mapped.ERROR("删除失败"); } @Async Future countTotalByModelId(String modelId, String generated) { Criteria criteria = Criteria.where("modelId").is(modelId); if ("true".equals(generated)) { criteria.and("generated").is(Boolean.TRUE); } else if ("false".equals(generated)) { criteria.and("generated").is(Boolean.FALSE); } Long total = this.mongoTemplate.count(new Query(criteria), LogicalFile.class); return new AsyncResult(total); } @Async public void removePhysicalFile(String id) { this.fsOperations.delete(new Query(Criteria.where("_id").is(id))); } @Async public void removeLogicalFile(String fileId) { this.mongoTemplate.remove(new Query(Criteria.where("id").is(fileId)), LogicalFile.class); } @Async public void removeDataByFileId(String fileId) { this.mongoTemplate.remove(new Query(Criteria.where("fileId").is(fileId)), DataMap.class); } @Async public void removeFieldsByFileId(String fileId) { this.mongoTemplate.remove(new Query(Criteria.where("fileId").is(fileId)), Fields.class); } @Async Future retrieveModelname(String modelId) { Query query = new Query(Criteria.where("id").is(modelId)); query.fields().include("modelName"); Model model = this.mongoTemplate.findOne(query, Model.class, "model"); return new AsyncResult(model != null ? model.getModelName() : ""); } }