// // Source code recreated from a .class file by IntelliJ IDEA // (powered by FernFlower decompiler) // package ieven.server.webapp.domain.file; import cn.hutool.core.date.DateUtil; import cn.hutool.core.io.FileUtil; import cn.hutool.core.io.IoUtil; import cn.hutool.core.util.IdUtil; import cn.hutool.poi.excel.ExcelReader; import cn.hutool.poi.excel.ExcelUtil; import com.mongodb.client.gridfs.model.GridFSFile; import com.mongodb.client.result.DeleteResult; import com.opencsv.CSVReader; import ieven.server.webapp.domain.IdInput; import ieven.server.webapp.domain.data.DataMap; import ieven.server.webapp.domain.data.Fields; import ieven.server.webapp.domain.model.Model; import ieven.server.webapp.domain.tenpay.TenpayHeaders; import ieven.server.webapp.infrastructure.wrapper.Mapped; import ieven.server.webapp.service.MongoExcelService; import ieven.server.webapp.util.EncodeDetector; import ieven.server.webapp.util.FileUtils; import ieven.server.webapp.util.excel.ExcelXlsReader; import ieven.server.webapp.util.excel.ExcelXlsxReader; import ieven.server.webapp.util.excel.PublicStatic; import info.monitorenter.cpdetector.io.*; import lombok.extern.slf4j.Slf4j; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; import org.apache.commons.lang3.StringUtils; import org.apache.poi.ss.usermodel.Sheet; import org.bson.types.ObjectId; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; import org.springframework.data.domain.Sort; import org.springframework.data.domain.Sort.Order; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Update; import org.springframework.data.mongodb.gridfs.GridFsOperations; import org.springframework.data.mongodb.gridfs.GridFsResource; import org.springframework.data.mongodb.gridfs.GridFsTemplate; import org.springframework.scheduling.annotation.Async; import org.springframework.scheduling.annotation.AsyncResult; import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; import java.io.*; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.concurrent.*; import java.util.regex.Pattern; import java.util.stream.Collectors; import static cn.hutool.core.util.ArrayUtil.contains; @Service @EnableAsync @Slf4j public class FileService { @Value("${spring.data.fileExportPath}") private String fileUploadPath; String[] order1 = {"反诈数据-订单明细", "反诈数据-订单明细2", "治安数据-订单明细", "经侦数据-订单明细", "网黑数据-订单明细", "反诈治安-交易明细", "反诈治安-账户透视", "反诈治安-对手透视"}; String[] order2 = { "五联单-注册信息", "经侦云-注册信息", "五联单-登录日志", "五联单-交易记录", "五联单-账户明细", "经侦云-账户明细", "五联单-转账明细", "五联单-转账分析", "五联单-账户透视", "五联单-对手透视", "五联单-IP地址分析", "五联单-收货地址分析" }; String[] order3 = {"财付通-注册信息", "财付通-订单明细1", "财付通-订单明细2", "财付通-商户注册信息","财付通-商户订单", "财付通-交易明细", "财付通-账户透视", "财付通-对手透视", "财付通-手机明细"}; String[] order4 = {"数据分析-主体信息汇总", "数据分析-账户透视", "数据分析-对手透视", "数据分析-共同关系网络"}; @Autowired private GridFsTemplate gridFsTemplate; @Autowired private MongoTemplate mongoTemplate; @Autowired private GridFsOperations fsOperations; @Autowired @Lazy private FileService fileService; public FileService() { } public static String getCharsetName(File file) throws IOException { String charsetName = "UTF-8"; // 获取 CodepageDetectorProxy 实例 CodepageDetectorProxy detector = CodepageDetectorProxy.getInstance(); // 添加解析器,会使用到添加的后 2 个 ext 里的 jar 包 detector.add(new ParsingDetector(false)); detector.add(JChardetFacade.getInstance()); detector.add(ASCIIDetector.getInstance()); detector.add(UnicodeDetector.getInstance()); Charset charset = detector.detectCodepage(file.toURI().toURL()); if (charset != null) { charsetName = charset.name(); } return charsetName; } public static void main(String[] args) throws IOException { File file = new File( "C:\\Users\\Administrator\\Desktop\\楼\\支付宝五联单数据格式\\20120303组织卖淫案_OR222713D2400411_注册信息_20210913100124.csv"); String charsetName = getCharsetName(file); CSVParser csvParser = CSVFormat.EXCEL.parse(new InputStreamReader(new FileInputStream(file), charsetName)); for (CSVRecord strings : csvParser) { System.out.println(strings); } file = new File("C:\\Users\\Administrator\\Desktop\\楼\\微信支付账单(2).csv"); charsetName = getCharsetName(file); csvParser = CSVFormat.EXCEL.parse(new InputStreamReader(new FileInputStream(file), charsetName)); for (CSVRecord strings : csvParser) { System.out.println(strings); } } public Mapped storeUploaded( InputStream inputStream, String encoding, String filename, String contentType, String modelId) { ObjectId objectId; objectId = this.gridFsTemplate.store(inputStream, filename, contentType); Query query = new Query( Criteria.where("modelId") .is(modelId) .and("filename") .is(filename)); List existsFileList = this.mongoTemplate.find(query, LogicalFile.class); if (existsFileList != null && existsFileList.size() > 0) { return Mapped.OK(); } LogicalFile logicalFile = new LogicalFile(); logicalFile.setFilename(filename); logicalFile.setUploadDate(DateUtil.now()); logicalFile.setModelId(modelId); logicalFile.setGridId(objectId); logicalFile.setGenerated(Boolean.FALSE); logicalFile.setStatus("LOADING"); logicalFile = this.mongoTemplate.insert(logicalFile); this.readFile(logicalFile, encoding); return Mapped.OK(); } @Async public void readFile(LogicalFile uploaded, String encoding) { GridFSFile fsFile = this.fsOperations.findOne(new Query(Criteria.where("_id").is(uploaded.getGridId()))); if (fsFile == null) { this.modifyStatus(uploaded.getId(), "ERROR"); } else { GridFsResource resource = this.fsOperations.getResource(fsFile); String filename = resource.getFilename(); if (StringUtils.isBlank(filename)) { this.modifyStatus(uploaded.getId(), "ERROR"); } else { try { InputStream inputStream = resource.getInputStream(); MongoExcelService mongoExcelService; if (filename.endsWith(".xlsx")) { mongoExcelService = new MongoExcelService(this.mongoTemplate, uploaded.getId()); ExcelXlsxReader reader = new ExcelXlsxReader(mongoExcelService); reader.processStream(inputStream); mongoExcelService.insertRest(); } else if (filename.endsWith(".xls")) { mongoExcelService = new MongoExcelService(this.mongoTemplate, uploaded.getId()); ExcelXlsReader reader = new ExcelXlsReader(mongoExcelService); reader.processStream(inputStream); mongoExcelService.insertRest(); } else if (filename.endsWith(".csv")) { mongoExcelService = new MongoExcelService(this.mongoTemplate, uploaded.getId()); log.info("encoding2:{}", encoding); /*CSVParser csvParser = null; if (encoding.equalsIgnoreCase("UTF-8")) { csvParser = CSVFormat.EXCEL.parse(new InputStreamReader(inputStream, "utf-8")); } else { csvParser = CSVFormat.EXCEL.parse(new InputStreamReader(inputStream)); }*/ int curRow = 0; int curRowWxFile = 0; boolean wxFile = false; String nickname = ""; // 微信手机账单文件 添加一列 微信昵称 需要单独处理 try (CSVReader reader = new CSVReader(new InputStreamReader(inputStream))) { List lines = reader.readAll(); for (String[] line : lines) { log.info("record,{}", line); List strings = Arrays.asList(line); List strings2 = new ArrayList<>(); strings2.addAll(strings); /************* 此处内容一下均为处理微信账单表头 ****************/ String join = String.join("", strings); join = PublicStatic.removeAllSpecial(join); if (join.equals("微信支付账单明细")) { wxFile = true; } if (wxFile && curRowWxFile < 16) { if (curRowWxFile == 1) { String temp = strings.get(0); if (StringUtils.isNotBlank(temp)) { nickname = temp.replace("微信昵称:[", "").replace("]", ""); } } curRowWxFile++; continue; } if (wxFile && curRowWxFile > 16) { strings2.add(0, nickname); curRowWxFile++; } if (wxFile && curRowWxFile == 16) { strings2.add(0, "微信昵称"); curRowWxFile++; } /************* 此处内容以上均为处理微信账单表头 ****************/ mongoExcelService.getRows( 0, curRow, strings2.stream() .map(PublicStatic::removeAllSpecial) .collect(Collectors.toList()) ); curRow++; } } catch (Exception e) { e.printStackTrace(); } mongoExcelService.insertRest(); } else if (filename.endsWith(".txt")) { mongoExcelService = new MongoExcelService(this.mongoTemplate, uploaded.getId()); List lines = new ArrayList<>(); IoUtil.readLines(new InputStreamReader(inputStream, "utf-8"), lines); int curRow = 0; boolean processNext = false; boolean lineAccount = false; for (String line : lines) { if (line.equals("注销信息")) { continue; } if (processNext) { List values = Arrays.asList(line.split("\t")); if (values.size() > 1) { mongoExcelService.getRows( 0, curRow, values.stream().map(PublicStatic::removeAllSpecial).collect(Collectors.toList()) ); curRow++; } break; } /* * // 定义需要检查的字段 String[] requiredFields = {"账户状态", "账号", "注册姓名", "注册时间", "注册身份证号"}; // 将行拆分为字段数组 String[] fields = line.split("\t"); // 检查每个必要字段是否存在 for (String field : requiredFields) { if (contains(fields, field) && !lineAccount) { lineAccount = true; } } * */ if (line.contains("账户状态\t账号\t注册姓名\t注册时间\t注册身份证号\t绑定手机\t绑定状态")) { // 先存储表头行 mongoExcelService.getRows( 0, curRow, Arrays.stream(line.split("\t")) .map(PublicStatic::removeAllSpecial) .collect(Collectors.toList()) ); curRow++; processNext = true; } } int isMerchant = 0; // 商户注册信息 HashMap merchantMap = new HashMap<>(); List headers = TenpayHeaders.getHeaders(11); for (String line : lines) { try { for (String header : headers) { if (line.contains(header)) { // Split each line into left and right parts String[] merchantLine = line.split(":"); String key = merchantLine[0].trim(); String value = merchantLine.length < 2 ? "" : merchantLine[1].trim(); merchantMap.put(key, value); isMerchant++; } } } catch (ArrayIndexOutOfBoundsException e) { System.out.println("错误: 行解析失败,行内容可能不正确。"); System.out.println("错误信息: " + e.getMessage()); e.printStackTrace(); } catch (NullPointerException e) { System.out.println("错误: 空指针异常,可能是headers或line为空。"); System.out.println("错误信息: " + e.getMessage()); e.printStackTrace(); } } System.out.println(merchantMap); /*财付通-商户注册信息 */ if(merchantMap.size() > 8) { String bankNo = merchantMap.get("绑定银行卡号"); /* * "商户账号", "商户名", "注册时间", "证件号", "邮箱", "绑定手机", "银行账户名", "绑定银行", "绑定银行卡号", "商户地址" * */ String[] bankNoArray = bankNo.split(","); mongoExcelService.getRows( 0, 0, headers ); for (String no : bankNoArray) { // 创建每行的数据,包含所有字段,其中bankNo为当前值 List row = new ArrayList<>(); row.add(merchantMap.get("商户账号")); row.add(merchantMap.get("商户名")); row.add(merchantMap.get("注册时间")); row.add(merchantMap.get("证件号")); row.add(merchantMap.get("邮箱")); row.add(merchantMap.get("绑定手机")); row.add(merchantMap.get("银行账户名")); row.add(merchantMap.get("绑定银行")); row.add(no); row.add(merchantMap.get("商户地址")); // 假设bankNo是第三个字段 // 添加其他字段... mongoExcelService.getRows( 0, 1, row ); } } mongoExcelService.insertRest(); } } catch (Exception var11) { this.modifyStatus(uploaded.getId(), "ERROR"); var11.printStackTrace(); } this.modifyStatus(uploaded.getId(), "FINISHED"); } } } private void modifyStatus(String id, String status) { Query query = new Query(); query.addCriteria(Criteria.where("_id").is(id)); Update update = new Update(); update.set("status", status); this.mongoTemplate.updateFirst(query, update, "logical_file"); } public Mapped listGroupUploaded(FileListInput fileListInput) { String modelId = fileListInput.getModelId(); if (modelId == null) { return Mapped.ERROR("没有选择任何分组"); } else { Query query = new Query(Criteria.where("modelId").is(modelId).and("generated").is(Boolean.TRUE)); query.with(Sort.by(Order.desc("uploadDate"))); List resultList = this.mongoTemplate.find(query, LogicalFile.class); // 将文件 按照 反诈 财付通 五联单 进行分类 List groupLogincalFiles = new ArrayList<>(); groupLogicalFile("反诈治安数据分析", new String[]{"反诈", "治安", "经侦数据", "网黑"}, resultList, groupLogincalFiles); groupLogicalFile("支付宝数据分析", new String[]{"五联单", "经侦云"}, resultList, groupLogincalFiles); groupLogicalFile("财付通数据分析", new String[]{"财付通"}, resultList, groupLogincalFiles); groupLogicalFile("数据综合分析", new String[]{"数据分析"}, resultList, groupLogincalFiles); Mapped res = Mapped.OK(); res.put("code", 200); res.put("result", groupLogincalFiles); return res; } } public void groupLogicalFile( String groupName, String[] group, List logicalFileList, List groupLogincalFiles) { List children = new ArrayList<>(); for (String s : group) { children.addAll( logicalFileList.stream() .filter(t -> t.getFilename().startsWith(s)) .collect(Collectors.toList())); } if (children != null && children.size() > 0) { GroupLogincalFile groupLogincalFile = new GroupLogincalFile(); groupLogincalFile.setGroupName(groupName); groupLogincalFile.setId(IdUtil.randomUUID()); groupLogincalFile.setChildren(orderGroupLogicalFile(groupName, children)); groupLogincalFiles.add(groupLogincalFile); } } public List orderGroupLogicalFile(String groupName, List list) { if (groupName.equals("反诈治安数据分析")) { return orderGroupLogicalFileChild(order1, list); } if (groupName.equals("支付宝数据分析")) { return orderGroupLogicalFileChild(order2, list); } if (groupName.equals("财付通数据分析")) { return orderGroupLogicalFileChild(order3, list); } if (groupName.equals("数据综合分析")) { return orderGroupLogicalFileChild(order4, list); } return null; } public List orderGroupLogicalFileChild(String[] orders, List list) { List child = new ArrayList<>(); for (String s : orders) { for (LogicalFile logicalFile : list) { if (logicalFile.getFilename().equals(s)) { if ("数据分析-共同关系网络".equals(logicalFile.getFilename())) { logicalFile.setFileType("map"); } child.add(logicalFile); break; } } } return child; } public Mapped listUploaded(FileListInput fileListInput) { String modelId = fileListInput.getModelId(); if (modelId == null) { return Mapped.ERROR("没有选择任何分组"); } else { String generated = fileListInput.getGenerated(); String fileName = fileListInput.getFileName(); Future longFuture = this.fileService.countTotalByModelId(modelId, generated); Future modelNameFuture = this.fileService.retrieveModelname(modelId); int page = fileListInput.getPage(); int pageSize = fileListInput.getPageSize(); int skip = (page - 1) * pageSize; Criteria criteria = Criteria.where("modelId").is(modelId); if ("true".equals(generated)) { criteria.and("generated").is(Boolean.TRUE); } else if ("false".equals(generated)) { criteria.and("generated").is(Boolean.FALSE); } if (fileName != null && !fileName.isEmpty()) { // 使用正则表达式进行模糊匹配,i 表示不区分大小写 Pattern pattern = Pattern.compile(".*" + Pattern.quote(fileName) + ".*", Pattern.CASE_INSENSITIVE); criteria.and("filename").regex(pattern); } // 创建查询对象并执行查询 Query query = new Query(criteria) .with(Sort.by(Sort.Direction.DESC, "uploadDate")) // 保留排序逻辑 .skip(skip) .limit(pageSize); List resultList = this.mongoTemplate.find(query, LogicalFile.class); Long total; try { total = longFuture.get(); } catch (ExecutionException | InterruptedException var17) { return Mapped.ERROR("查询出错"); } String modelName; try { modelName = modelNameFuture.get(); } catch (ExecutionException | InterruptedException var17) { return Mapped.ERROR("查询出错"); } int i = skip; for (LogicalFile file : resultList) { ++i; file.setCount(i); file.setModelName(modelName); } Mapped res = Mapped.OK(); res.put("code", 200); res.put("result", resultList); res.put("total", total); res.put("page", page); res.put("pageSize", pageSize); return res; } } public Mapped deleteById(IdInput idInput) { String id = idInput.getId(); Query query = new Query(Criteria.where("id").is(id)); LogicalFile file = this.mongoTemplate.findOne(query, LogicalFile.class); if (file != null && file.getGridId() != null) { String physicalId = file.getGridId().toHexString(); this.fileService.removePhysicalFile(physicalId); } this.fileService.removeDataByFileId(id); this.fileService.removeFieldsByFileId(id); DeleteResult result = this.mongoTemplate.remove(query, LogicalFile.class); return result.getDeletedCount() > 0L ? Mapped.OK() : Mapped.ERROR("删除失败"); } @Async Future countTotalByModelId(String modelId, String generated) { Criteria criteria = Criteria.where("modelId").is(modelId); if ("true".equals(generated)) { criteria.and("generated").is(Boolean.TRUE); } else if ("false".equals(generated)) { criteria.and("generated").is(Boolean.FALSE); } Long total = this.mongoTemplate.count(new Query(criteria), LogicalFile.class); return new AsyncResult(total); } @Async public void removePhysicalFile(String id) { this.fsOperations.delete(new Query(Criteria.where("_id").is(id))); } @Async public void removeLogicalFile(String fileId) { this.mongoTemplate.remove(new Query(Criteria.where("id").is(fileId)), LogicalFile.class); } @Async public void removeDataByFileId(String fileId) { this.mongoTemplate.remove(new Query(Criteria.where("fileId").is(fileId)), DataMap.class); } @Async public void removeFieldsByFileId(String fileId) { this.mongoTemplate.remove(new Query(Criteria.where("fileId").is(fileId)), Fields.class); } @Async Future retrieveModelname(String modelId) { Query query = new Query(Criteria.where("id").is(modelId)); query.fields().include("modelName"); Model model = this.mongoTemplate.findOne(query, Model.class, "model"); return new AsyncResult(model != null ? model.getModelName() : ""); } public Mapped uploadFilesAndSave(List multipartFiles, String modelId) { List failNameList = new ArrayList<>(); for (int i = 0; i < multipartFiles.size(); i++) { MultipartFile multipartFile = multipartFiles.get(i); String originName = multipartFile.getOriginalFilename(); String suffix = originName.substring(originName.lastIndexOf(".") + 1); String prefix = originName.substring(0, originName.lastIndexOf(".")); String savedName = prefix + "(" + (i + 1) + ")." + suffix; String filepath = fileUploadPath + File.separator + modelId + File.separator + savedName; FileUtils.uploadFile(multipartFile, filepath); MultipartFile multipartFile1 = FileUtils.getMultipartFile(new File(filepath)); try { ExecutorService executor = Executors.newSingleThreadExecutor(); Future future = executor.submit(new Callable() { @Override public Mapped call() throws Exception { return uploadSingle(multipartFile1, modelId); } }); // 在这里,主线程会等待 future 的结果 Mapped mapped = future.get(); // 这将会阻塞,直到Callable任务完成 if (200 != Integer.valueOf(String.valueOf(mapped.get("code")))) { failNameList.add(savedName + ":" + String.valueOf(mapped.get("message"))); } //删除文件 FileUtils.deleteFile(filepath); } catch (Exception e) { e.printStackTrace(); } } Mapped mapped = Mapped.OK(); mapped.put("failNameList", failNameList); return mapped; } public Mapped uploadSingle(MultipartFile file, String modelId) throws Exception { String filename = file.getOriginalFilename(); String extName = FileUtil.extName(filename); String contentType = file.getContentType(); ByteArrayOutputStream baos = FileUtils.cloneInputStream(file.getInputStream()); if (extName.equalsIgnoreCase("xls") || extName.equalsIgnoreCase("xlsx")) { ExcelReader reader = ExcelUtil.getReader(new ByteArrayInputStream(baos.toByteArray())); List sheets = reader.getSheets(); List headers = new ArrayList<>(); for (Sheet sheet : sheets) { reader.setSheet(sheet); List objects = reader.readRow(0); if (objects.size() > 1) { // 至少大于1 StringBuilder stringBuilder = new StringBuilder(); for (Object object : objects) { stringBuilder.append(object); } headers.add(stringBuilder.toString()); } } if (headers.size() > 1) { // 判断sheet第一行内容是否一致 StringBuilder errorHeaders = new StringBuilder(); String baseHeader = headers.get(0); for (int i = 0; i < headers.size(); i++) { // if (i == 0) { continue; } if (!headers.get(i).equals(baseHeader)) { errorHeaders.append("sheet").append(i).append("标题列与第一个sheet不一致!"); } } if (errorHeaders.length() > 0) { return Mapped.ERROR(errorHeaders.toString()); } } reader.close(); } String encoding = EncodeDetector.getEncoding(baos.toByteArray()); baos.close(); return this.fileService.storeUploaded( new ByteArrayInputStream(baos.toByteArray()), encoding, filename, contentType, modelId); } }