财付通新的订单格式

pom.xml

@@ -184,6 +184,4 @@
             </snapshots>
         </pluginRepository>
     </pluginRepositories>
-
-
 </project>

src/main/java/ieven/server/webapp/api/AlipayController.java

@@ -33,361 +33,365 @@ import java.util.concurrent.Future;
 @RestController
 @RequestMapping({"/alipay"})
 public class AlipayController {
-  @Autowired private AlipayService alipayService;
-  @Autowired private MongoTemplate mongoTemplate;
-  @Autowired private DataService dataService;
-
-  public AlipayController() {}
-
-  @PostMapping({"/extractPersonInfo"})
-  public Mapped extractPersonInfo(@RequestBody ModelIdInput modelIdInput)
-      throws InterruptedException {
-    String modelId = modelIdInput.getModelId();
-    return modelId == null ? Mapped.ERROR("没有选择模型!") : this.alipayService.extractAll(modelId);
-  }
-
-  @PostMapping({"/viewDetailContent"})
-  public Mapped viewDetailContent(@RequestBody LineInput input) {
-    String modelId = input.getModelId();
-    if (modelId == null) {
-      return Mapped.ERROR("没有选择模型!");
-    } else {
-      String filename = input.getFilename();
-      String fileId = input.getFileId();
-      String clickHeader = input.getClickHeader();
-      String lineId = input.getLineId();
-      int page = input.getPage();
-      int pageSize = input.getPageSize();
-      int skip = (page - 1) * pageSize;
-      Criteria criteria = this.resolveDetailQuery(filename, fileId, lineId, clickHeader);
-      if (criteria == null) {
-        return Mapped.ERROR("未查询到明细");
-      } else {
-        Future<Long> totalFuture = this.dataService.countTotalByQuery(new Query(criteria));
-        List<DataMap> dataMaps =
-            this.mongoTemplate.find(
-                (new Query(criteria)).skip(skip).limit(pageSize), DataMap.class);
-        Long total;
-
-        try {
-          total = totalFuture.get();
-        } catch (ExecutionException | InterruptedException var15) {
-          var15.printStackTrace();
-          return Mapped.ERROR("系统查询出错");
-        }
-        Mapped mapped = Mapped.OK();
-        mapped.put("result", dataMaps);
-        mapped.put("code", 200);
-        mapped.put("page", page);
-        mapped.put("total", total);
-        mapped.put("fileId", fileId);
-        mapped.put("pageSize", pageSize);
-        return mapped;
-      }
-    }
-  }
-
-  @PostMapping({"/viewDetailHeaders"})
-  public Mapped viewHeaders(@RequestBody LineInput input) {
-    String modelId = input.getModelId();
-    if (modelId == null) {
-      return Mapped.ERROR("没有选择模型!");
-    } else {
-      String fileId = input.getFileId();
-      TFromToFileIds oldFile =
-          this.mongoTemplate.findOne(
-              new Query(Criteria.where("toFileId").is(fileId)), TFromToFileIds.class);
-      if (oldFile == null) {
-        return Mapped.ERROR("未发现明细!");
-      } else {
-        String fromFileId = oldFile.getFromFileId();
-        Fields fields =
-            this.mongoTemplate.findOne(
-                new Query(Criteria.where("fileId").is(fromFileId)), Fields.class);
-        Mapped mapped = Mapped.OK();
-        mapped.put("result", fields);
-        return mapped;
-      }
+    @Autowired
+    private AlipayService alipayService;
+    @Autowired
+    private MongoTemplate mongoTemplate;
+    @Autowired
+    private DataService dataService;
+
+    public AlipayController() {
     }
-  }
-
-  private List<String> destFileId(String oriFileId) {
-    Set<String> destFileIds = new HashSet<>();
-    List<TFromToFileIds> oldFileList =
-        this.mongoTemplate.find(
-            new Query(Criteria.where("toFileId").is(oriFileId)), TFromToFileIds.class);
 
-    for (TFromToFileIds t : oldFileList) {
-      destFileIds.add(t.getFromFileId());
+    @PostMapping({"/extractPersonInfo"})
+    public Mapped extractPersonInfo(@RequestBody ModelIdInput modelIdInput)
+            throws InterruptedException {
+        String modelId = modelIdInput.getModelId();
+        return modelId == null ? Mapped.ERROR("没有选择模型!") : this.alipayService.extractAll(modelId);
     }
 
-    return new ArrayList<>(destFileIds);
-  }
-
-  @PostMapping({"/exportViewDetailContent"})
-  public Mapped exportViewDetailContent(@RequestBody LineInput input) throws IOException {
-    String modelId = input.getModelId();
-    if (modelId == null) {
-      return Mapped.ERROR("没有选择模型!");
-    } else {
-      String filename = input.getFilename();
-      String fileId = input.getFileId();
-      String clickHeader = input.getClickHeader();
-      String lineId = input.getLineId();
-      Criteria criteria = this.resolveDetailQuery(filename, fileId, lineId, clickHeader);
-      if (criteria == null) {
-        return Mapped.ERROR("未查询到明细");
-      } else {
-        MongoCursor<Document> cursor =
-            this.mongoTemplate
-                .getCollection("data")
-                .find((new Query(criteria)).getQueryObject())
-                .batchSize(1000)
-                .noCursorTimeout(true)
-                .cursor();
-        Exporter exporter = new Exporter();
-        List<String> fileIds = this.destFileId(fileId);
-        String exportFileId = exporter.export(cursor, fileIds.get(0), null);
-        Mapped mapped = Mapped.OK();
-        mapped.put("result", exportFileId);
-        return mapped;
-      }
-    }
-  }
-
-  @PostMapping({"/exportAllDetail"})
-  public Mapped exportAllDetail(@RequestBody LineInput input) throws IOException {
-    String modelId = input.getModelId();
-    if (modelId == null) {
-      return Mapped.ERROR("没有选择模型!");
-    } else {
-      String filename = input.getFilename();
-      String fileId = input.getFileId();
-      Criteria criteria = this.resolveDetailQuery(filename, fileId, null, null);
-      if (criteria == null) {
-        return Mapped.ERROR("未查询到明细");
-      } else {
-        MongoCursor<Document> cursor =
-            this.mongoTemplate
-                .getCollection("data")
-                .find((new Query(criteria)).getQueryObject())
-                .batchSize(1000)
-                .noCursorTimeout(true)
-                .cursor();
-        Exporter exporter = new Exporter();
-        List<String> fileIds = this.destFileId(fileId);
-        if (StringUtils.isNotBlank(modelId) && filename == null) {
-          Model model =
-              mongoTemplate.findOne(new Query(Criteria.where("id").is(modelId)), Model.class);
-          filename = model.getModelName();
+    @PostMapping({"/viewDetailContent"})
+    public Mapped viewDetailContent(@RequestBody LineInput input) {
+        String modelId = input.getModelId();
+        if (modelId == null) {
+            return Mapped.ERROR("没有选择模型!");
+        } else {
+            String filename = input.getFilename();
+            String fileId = input.getFileId();
+            String clickHeader = input.getClickHeader();
+            String lineId = input.getLineId();
+            int page = input.getPage();
+            int pageSize = input.getPageSize();
+            int skip = (page - 1) * pageSize;
+            Criteria criteria = this.resolveDetailQuery(filename, fileId, lineId, clickHeader);
+            if (criteria == null) {
+                return Mapped.ERROR("未查询到明细");
+            } else {
+                Future<Long> totalFuture = this.dataService.countTotalByQuery(new Query(criteria));
+                List<DataMap> dataMaps =
+                        this.mongoTemplate.find(
+                                (new Query(criteria)).skip(skip).limit(pageSize), DataMap.class);
+                Long total;
+
+                try {
+                    total = totalFuture.get();
+                } catch (ExecutionException | InterruptedException var15) {
+                    var15.printStackTrace();
+                    return Mapped.ERROR("系统查询出错");
+                }
+                Mapped mapped = Mapped.OK();
+                mapped.put("result", dataMaps);
+                mapped.put("code", 200);
+                mapped.put("page", page);
+                mapped.put("total", total);
+                mapped.put("fileId", fileId);
+                mapped.put("pageSize", pageSize);
+                return mapped;
+            }
         }
-        String exportFileId = exporter.export(cursor, fileIds.get(0), filename);
-        Mapped mapped = Mapped.OK();
-        mapped.put("result", exportFileId);
-        return mapped;
-      }
-    }
-  }
-
-  private Criteria resolveDetailQuery(
-      String filename, String fileId, String lineId, String clickHeader) {
-    List<String> fileIds = this.destFileId(fileId);
-    Criteria criteria = Criteria.where("fileId");
-    if (fileIds.size() == 1) {
-      criteria.is(fileIds.get(0));
-    } else if (fileIds.size() > 1) {
-      criteria.in(fileIds);
-    } else {
-      return null;
     }
 
-    DataMap dataMap;
-    String dszfbyhid;
-    if (filename.equals("反诈治安-账户透视")) {
-      if (lineId == null) {
-        return criteria;
-      }
-
-      dataMap = this.mongoTemplate.findById(lineId, DataMap.class);
-      if (dataMap == null) {
-        return null;
-      }
-
-      dszfbyhid = (String) dataMap.get("交易主体账号");
-      if (!clickHeader.equals("交易总额") && !clickHeader.equals("总笔数")) {
-        if (!clickHeader.equals("收入金额") && !clickHeader.equals("收入笔数")) {
-          if (clickHeader.equals("支出金额") || clickHeader.equals("支出笔数")) {
-            criteria.and("交易主体账号").is(dszfbyhid).and("交易主体的出入账标识").is("出账");
-          }
+    @PostMapping({"/viewDetailHeaders"})
+    public Mapped viewHeaders(@RequestBody LineInput input) {
+        String modelId = input.getModelId();
+        if (modelId == null) {
+            return Mapped.ERROR("没有选择模型!");
         } else {
-          criteria.and("交易主体账号").is(dszfbyhid).and("交易主体的出入账标识").is("入账");
+            String fileId = input.getFileId();
+            TFromToFileIds oldFile =
+                    this.mongoTemplate.findOne(
+                            new Query(Criteria.where("toFileId").is(fileId)), TFromToFileIds.class);
+            if (oldFile == null) {
+                return Mapped.ERROR("未发现明细!");
+            } else {
+                String fromFileId = oldFile.getFromFileId();
+                Fields fields =
+                        this.mongoTemplate.findOne(
+                                new Query(Criteria.where("fileId").is(fromFileId)), Fields.class);
+                Mapped mapped = Mapped.OK();
+                mapped.put("result", fields);
+                return mapped;
+            }
         }
-      } else {
-        criteria.and("交易主体账号").is(dszfbyhid);
-      }
-    } else if (filename.equals("反诈治安-对手透视")) {
-      if (lineId == null) {
-        return criteria;
-      }
-
-      dataMap = this.mongoTemplate.findById(lineId, DataMap.class);
-      if (dataMap == null) {
-        return null;
-      }
-
-      dszfbyhid = (String) dataMap.get("交易对手账号");
-      if (!clickHeader.equals("交易总额") && !clickHeader.equals("总笔数")) {
-        if (!clickHeader.equals("交易主体收入金额") && !clickHeader.equals("交易主体收入笔数")) {
-          if (clickHeader.equals("交易主体支出金额") || clickHeader.equals("交易主体支出笔数")) {
-            criteria.and("交易对手账号").is(dszfbyhid).and("交易主体的出入账标识").is("出账");
-          }
-        } else {
-          criteria.and("交易对手账号").is(dszfbyhid).and("交易主体的出入账标识").is("入账");
-        }
-      } else {
-        criteria.and("交易对手账号").is(dszfbyhid);
-      }
-    } else if (filename.equals("财付通-账户透视")) {
-      if (lineId == null) {
-        return criteria;
-      }
-
-      dataMap = this.mongoTemplate.findById(lineId, DataMap.class);
-      if (dataMap == null) {
-        return null;
-      }
-
-      dszfbyhid = (String) dataMap.get("交易主体账号");
-      if (!clickHeader.equals("交易总额") && !clickHeader.equals("总笔数")) {
-        if (!clickHeader.equals("收入金额") && !clickHeader.equals("收入笔数")) {
-          if (clickHeader.equals("支出金额") || clickHeader.equals("支出笔数")) {
-            criteria.and("交易主体账号").is(dszfbyhid).and("借贷类型").is("出");
-          }
-        } else {
-          criteria.and("交易主体账号").is(dszfbyhid).and("借贷类型").is("入");
-        }
-      } else {
-        criteria.and("交易主体账号").is(dszfbyhid);
-      }
-    } else if (filename.equals("财付通-对手透视")) {
-      if (lineId == null) {
-        return criteria;
-      }
-
-      dataMap = this.mongoTemplate.findById(lineId, DataMap.class);
-      if (dataMap == null) {
-        return null;
-      }
-
-      dszfbyhid = (String) dataMap.get("交易对手账号");
-      if (!clickHeader.equals("交易总额") && !clickHeader.equals("总笔数")) {
-        if (!clickHeader.equals("交易主体收入金额") && !clickHeader.equals("交易主体收入笔数")) {
-          if (clickHeader.equals("交易主体支出金额") || clickHeader.equals("交易主体支出笔数")) {
-            criteria.and("交易对手账号").is(dszfbyhid).and("借贷类型").is("出");
-          }
-        } else {
-          criteria.and("交易对手账号").is(dszfbyhid).and("借贷类型").is("入");
+    }
+
+    private List<String> destFileId(String oriFileId) {
+        Set<String> destFileIds = new HashSet<>();
+        List<TFromToFileIds> oldFileList =
+                this.mongoTemplate.find(
+                        new Query(Criteria.where("toFileId").is(oriFileId)), TFromToFileIds.class);
+
+        for (TFromToFileIds t : oldFileList) {
+            destFileIds.add(t.getFromFileId());
         }
-      } else {
-        criteria.and("交易对手账号").is(dszfbyhid);
-      }
-    } else if (filename.equals("五联单-转账分析")) {
-      if (lineId == null) {
-        criteria.and("消费名称").is("转账");
-        return criteria;
-      }
-
-      dataMap = this.mongoTemplate.findById(lineId, DataMap.class);
-      if (dataMap == null) {
-        return null;
-      }
-
-      dszfbyhid = (String) dataMap.get("支付宝用户ID");
-      if (!clickHeader.equals("转账总额") && !clickHeader.equals("总笔数")) {
-        if (!clickHeader.equals("转账收入") && !clickHeader.equals("转收笔数")) {
-          if (clickHeader.equals("转账支出") || clickHeader.equals("转支笔数")) {
-            criteria.and("支付宝用户ID").is(dszfbyhid).and("消费名称").is("转账").and("收/支").is("支出");
-          }
+
+        return new ArrayList<>(destFileIds);
+    }
+
+    @PostMapping({"/exportViewDetailContent"})
+    public Mapped exportViewDetailContent(@RequestBody LineInput input) throws IOException {
+        String modelId = input.getModelId();
+        if (modelId == null) {
+            return Mapped.ERROR("没有选择模型!");
         } else {
-          criteria.and("支付宝用户ID").is(dszfbyhid).and("消费名称").is("转账").and("收/支").is("收入");
+            String filename = input.getFilename();
+            String fileId = input.getFileId();
+            String clickHeader = input.getClickHeader();
+            String lineId = input.getLineId();
+            Criteria criteria = this.resolveDetailQuery(filename, fileId, lineId, clickHeader);
+            if (criteria == null) {
+                return Mapped.ERROR("未查询到明细");
+            } else {
+                MongoCursor<Document> cursor =
+                        this.mongoTemplate
+                                .getCollection("data")
+                                .find((new Query(criteria)).getQueryObject())
+                                .batchSize(1000)
+                                .noCursorTimeout(true)
+                                .cursor();
+                Exporter exporter = new Exporter();
+                List<String> fileIds = this.destFileId(fileId);
+                String exportFileId = exporter.export(cursor, fileIds.get(0), null);
+                Mapped mapped = Mapped.OK();
+                mapped.put("result", exportFileId);
+                return mapped;
+            }
         }
-      } else {
-        criteria.and("支付宝用户ID").is(dszfbyhid).and("消费名称").is("转账");
-      }
-    } else if (filename.equals("五联单-账户透视")) {
-      if (lineId == null) {
-        return criteria;
-      }
-
-      dataMap = this.mongoTemplate.findById(lineId, DataMap.class);
-      if (dataMap == null) {
-        return null;
-      }
-
-      dszfbyhid = (String) dataMap.get("支付宝用户ID");
-      if (!clickHeader.equals("交易总额") && !clickHeader.equals("总笔数")) {
-        if (!clickHeader.equals("收入金额") && !clickHeader.equals("收入笔数")) {
-          if (clickHeader.equals("支出金额") || clickHeader.equals("支出笔数")) {
-            criteria.and("支付宝用户ID").is(dszfbyhid).and("收/支").is("支出");
-          }
+    }
+
+    @PostMapping({"/exportAllDetail"})
+    public Mapped exportAllDetail(@RequestBody LineInput input) throws IOException {
+        String modelId = input.getModelId();
+        if (modelId == null) {
+            return Mapped.ERROR("没有选择模型!");
         } else {
-          criteria.and("支付宝用户ID").is(dszfbyhid).and("收/支").is("收入");
+            String filename = input.getFilename();
+            String fileId = input.getFileId();
+            Criteria criteria = this.resolveDetailQuery(filename, fileId, null, null);
+            if (criteria == null) {
+                return Mapped.ERROR("未查询到明细");
+            } else {
+                MongoCursor<Document> cursor =
+                        this.mongoTemplate
+                                .getCollection("data")
+                                .find((new Query(criteria)).getQueryObject())
+                                .batchSize(1000)
+                                .noCursorTimeout(true)
+                                .cursor();
+                Exporter exporter = new Exporter();
+                List<String> fileIds = this.destFileId(fileId);
+                if (StringUtils.isNotBlank(modelId) && filename == null) {
+                    Model model =
+                            mongoTemplate.findOne(new Query(Criteria.where("id").is(modelId)), Model.class);
+                    filename = model.getModelName();
+                }
+                String exportFileId = exporter.export(cursor, fileIds.get(0), filename);
+                Mapped mapped = Mapped.OK();
+                mapped.put("result", exportFileId);
+                return mapped;
+            }
         }
-      } else {
-        criteria.and("支付宝用户ID").is(dszfbyhid);
-      }
-    } else if (filename.equals("五联单-对手透视")) {
-      if (lineId == null) {
-        return criteria;
-      }
-
-      dataMap = this.mongoTemplate.findById(lineId, DataMap.class);
-      if (dataMap == null) {
-        return null;
-      }
-
-      dszfbyhid = (String) dataMap.get("对手支付宝用户ID");
-      if (!clickHeader.equals("交易总额") && !clickHeader.equals("总笔数")) {
-        if (!clickHeader.equals("主用户ID收入金额") && !clickHeader.equals("主用户ID收入笔数")) {
-          if (clickHeader.equals("主用户ID支出金额") || clickHeader.equals("主用户ID支出笔数")) {
-            criteria.and("对手支付宝用户ID").is(dszfbyhid).and("收/支").is("支出");
-          }
+    }
+
+    private Criteria resolveDetailQuery(
+            String filename, String fileId, String lineId, String clickHeader) {
+        List<String> fileIds = this.destFileId(fileId);
+        Criteria criteria = Criteria.where("fileId");
+        if (fileIds.size() == 1) {
+            criteria.is(fileIds.get(0));
+        } else if (fileIds.size() > 1) {
+            criteria.in(fileIds);
         } else {
-          criteria.and("对手支付宝用户ID").is(dszfbyhid).and("收/支").is("收入");
+            return null;
         }
-      } else {
-        criteria.and("对手支付宝用户ID").is(dszfbyhid);
-      }
-    } else if (filename.equals("五联单-收货地址分析")) {
-      if (lineId == null) {
-        return criteria;
-      }
-
-      dataMap = this.mongoTemplate.findById(lineId, DataMap.class);
-      if (dataMap == null) {
-        return null;
-      }
-
-      dszfbyhid = (String) dataMap.get("收货人地址");
-      if (clickHeader.equals("交易次数")) {
-        criteria.and("收货人地址").is(dszfbyhid);
-      }
-    } else if (filename.equals("五联单-IP地址分析")) {
-      if (lineId == null) {
-        return criteria;
-      }
 
-      dataMap = this.mongoTemplate.findById(lineId, DataMap.class);
-      if (dataMap == null) {
-        return null;
-      }
+        DataMap dataMap;
+        String dszfbyhid;
+        if (filename.equals("反诈治安-账户透视")) {
+            if (lineId == null) {
+                return criteria;
+            }
+
+            dataMap = this.mongoTemplate.findById(lineId, DataMap.class);
+            if (dataMap == null) {
+                return null;
+            }
+
+            dszfbyhid = (String) dataMap.get("交易主体账号");
+            if (!clickHeader.equals("交易总额") && !clickHeader.equals("总笔数")) {
+                if (!clickHeader.equals("收入金额") && !clickHeader.equals("收入笔数")) {
+                    if (clickHeader.equals("支出金额") || clickHeader.equals("支出笔数")) {
+                        criteria.and("交易主体账号").is(dszfbyhid).and("交易主体的出入账标识").is("出账");
+                    }
+                } else {
+                    criteria.and("交易主体账号").is(dszfbyhid).and("交易主体的出入账标识").is("入账");
+                }
+            } else {
+                criteria.and("交易主体账号").is(dszfbyhid);
+            }
+        } else if (filename.equals("反诈治安-对手透视")) {
+            if (lineId == null) {
+                return criteria;
+            }
+
+            dataMap = this.mongoTemplate.findById(lineId, DataMap.class);
+            if (dataMap == null) {
+                return null;
+            }
+
+            dszfbyhid = (String) dataMap.get("交易对手账号");
+            if (!clickHeader.equals("交易总额") && !clickHeader.equals("总笔数")) {
+                if (!clickHeader.equals("交易主体收入金额") && !clickHeader.equals("交易主体收入笔数")) {
+                    if (clickHeader.equals("交易主体支出金额") || clickHeader.equals("交易主体支出笔数")) {
+                        criteria.and("交易对手账号").is(dszfbyhid).and("交易主体的出入账标识").is("出账");
+                    }
+                } else {
+                    criteria.and("交易对手账号").is(dszfbyhid).and("交易主体的出入账标识").is("入账");
+                }
+            } else {
+                criteria.and("交易对手账号").is(dszfbyhid);
+            }
+        } else if (filename.equals("财付通-账户透视")) {
+            if (lineId == null) {
+                return criteria;
+            }
+
+            dataMap = this.mongoTemplate.findById(lineId, DataMap.class);
+            if (dataMap == null) {
+                return null;
+            }
+
+            dszfbyhid = (String) dataMap.get("交易主体账号");
+            if (!clickHeader.equals("交易总额") && !clickHeader.equals("总笔数")) {
+                if (!clickHeader.equals("收入金额") && !clickHeader.equals("收入笔数")) {
+                    if (clickHeader.equals("支出金额") || clickHeader.equals("支出笔数")) {
+                        criteria.and("交易主体账号").is(dszfbyhid).and("借贷类型").is("出");
+                    }
+                } else {
+                    criteria.and("交易主体账号").is(dszfbyhid).and("借贷类型").is("入");
+                }
+            } else {
+                criteria.and("交易主体账号").is(dszfbyhid);
+            }
+        } else if (filename.equals("财付通-对手透视")) {
+            if (lineId == null) {
+                return criteria;
+            }
+
+            dataMap = this.mongoTemplate.findById(lineId, DataMap.class);
+            if (dataMap == null) {
+                return null;
+            }
+
+            dszfbyhid = (String) dataMap.get("交易对手账号");
+            if (!clickHeader.equals("交易总额") && !clickHeader.equals("总笔数")) {
+                if (!clickHeader.equals("交易主体收入金额") && !clickHeader.equals("交易主体收入笔数")) {
+                    if (clickHeader.equals("交易主体支出金额") || clickHeader.equals("交易主体支出笔数")) {
+                        criteria.and("交易对手账号").is(dszfbyhid).and("借贷类型").is("出");
+                    }
+                } else {
+                    criteria.and("交易对手账号").is(dszfbyhid).and("借贷类型").is("入");
+                }
+            } else {
+                criteria.and("交易对手账号").is(dszfbyhid);
+            }
+        } else if (filename.equals("五联单-转账分析")) {
+            if (lineId == null) {
+                criteria.and("消费名称").is("转账");
+                return criteria;
+            }
+
+            dataMap = this.mongoTemplate.findById(lineId, DataMap.class);
+            if (dataMap == null) {
+                return null;
+            }
+
+            dszfbyhid = (String) dataMap.get("支付宝用户ID");
+            if (!clickHeader.equals("转账总额") && !clickHeader.equals("总笔数")) {
+                if (!clickHeader.equals("转账收入") && !clickHeader.equals("转收笔数")) {
+                    if (clickHeader.equals("转账支出") || clickHeader.equals("转支笔数")) {
+                        criteria.and("支付宝用户ID").is(dszfbyhid).and("消费名称").is("转账").and("收/支").is("支出");
+                    }
+                } else {
+                    criteria.and("支付宝用户ID").is(dszfbyhid).and("消费名称").is("转账").and("收/支").is("收入");
+                }
+            } else {
+                criteria.and("支付宝用户ID").is(dszfbyhid).and("消费名称").is("转账");
+            }
+        } else if (filename.equals("五联单-账户透视")) {
+            if (lineId == null) {
+                return criteria;
+            }
+
+            dataMap = this.mongoTemplate.findById(lineId, DataMap.class);
+            if (dataMap == null) {
+                return null;
+            }
+
+            dszfbyhid = (String) dataMap.get("支付宝用户ID");
+            if (!clickHeader.equals("交易总额") && !clickHeader.equals("总笔数")) {
+                if (!clickHeader.equals("收入金额") && !clickHeader.equals("收入笔数")) {
+                    if (clickHeader.equals("支出金额") || clickHeader.equals("支出笔数")) {
+                        criteria.and("支付宝用户ID").is(dszfbyhid).and("收/支").is("支出");
+                    }
+                } else {
+                    criteria.and("支付宝用户ID").is(dszfbyhid).and("收/支").is("收入");
+                }
+            } else {
+                criteria.and("支付宝用户ID").is(dszfbyhid);
+            }
+        } else if (filename.equals("五联单-对手透视")) {
+            if (lineId == null) {
+                return criteria;
+            }
+
+            dataMap = this.mongoTemplate.findById(lineId, DataMap.class);
+            if (dataMap == null) {
+                return null;
+            }
+
+            dszfbyhid = (String) dataMap.get("对手支付宝用户ID");
+            if (!clickHeader.equals("交易总额") && !clickHeader.equals("总笔数")) {
+                if (!clickHeader.equals("主用户ID收入金额") && !clickHeader.equals("主用户ID收入笔数")) {
+                    if (clickHeader.equals("主用户ID支出金额") || clickHeader.equals("主用户ID支出笔数")) {
+                        criteria.and("对手支付宝用户ID").is(dszfbyhid).and("收/支").is("支出");
+                    }
+                } else {
+                    criteria.and("对手支付宝用户ID").is(dszfbyhid).and("收/支").is("收入");
+                }
+            } else {
+                criteria.and("对手支付宝用户ID").is(dszfbyhid);
+            }
+        } else if (filename.equals("五联单-收货地址分析")) {
+            if (lineId == null) {
+                return criteria;
+            }
+
+            dataMap = this.mongoTemplate.findById(lineId, DataMap.class);
+            if (dataMap == null) {
+                return null;
+            }
+
+            dszfbyhid = (String) dataMap.get("收货人地址");
+            if (clickHeader.equals("交易次数")) {
+                criteria.and("收货人地址").is(dszfbyhid);
+            }
+        } else if (filename.equals("五联单-IP地址分析")) {
+            if (lineId == null) {
+                return criteria;
+            }
+
+            dataMap = this.mongoTemplate.findById(lineId, DataMap.class);
+            if (dataMap == null) {
+                return null;
+            }
+
+            dszfbyhid = (String) dataMap.get("IP地址");
+            if (clickHeader.equals("登录次数")) {
+                criteria.and("客户端IP").is(dszfbyhid);
+            }
+        }
 
-      dszfbyhid = (String) dataMap.get("IP地址");
-      if (clickHeader.equals("登录次数")) {
-        criteria.and("客户端IP").is(dszfbyhid);
-      }
+        return criteria;
     }
-
-    return criteria;
-  }
 }

src/main/java/ieven/server/webapp/domain/file/FileService.java

@@ -6,10 +6,8 @@
 package ieven.server.webapp.domain.file;
 
 import cn.hutool.core.date.DateUtil;
-import cn.hutool.core.io.FileUtil;
 import cn.hutool.core.io.IoUtil;
 import cn.hutool.core.util.IdUtil;
-import cn.hutool.core.util.RandomUtil;
 import com.mongodb.client.gridfs.model.GridFSFile;
 import com.mongodb.client.result.DeleteResult;
 import ieven.server.webapp.domain.IdInput;
@@ -18,7 +16,6 @@ import ieven.server.webapp.domain.data.Fields;
 import ieven.server.webapp.domain.model.Model;
 import ieven.server.webapp.infrastructure.wrapper.Mapped;
 import ieven.server.webapp.service.MongoExcelService;
-import ieven.server.webapp.util.EncodeDetector;
 import ieven.server.webapp.util.excel.ExcelXlsReader;
 import ieven.server.webapp.util.excel.ExcelXlsxReader;
 import ieven.server.webapp.util.excel.PublicStatic;
@@ -58,380 +55,384 @@ import java.util.stream.Collectors;
 @EnableAsync
 @Slf4j
 public class FileService {
-  @Autowired private GridFsTemplate gridFsTemplate;
-  @Autowired private MongoTemplate mongoTemplate;
-  @Autowired private GridFsOperations fsOperations;
-  @Autowired @Lazy private FileService fileService;
+    String[] order1 = {"反诈数据-订单明细", "治安数据-订单明细", "反诈治安-交易明细", "反诈治安-账户透视", "反诈治安-对手透视"};
+    String[] order2 = {
+            "五联单-注册信息",
+            "五联单-登录日志",
+            "五联单-交易记录",
+            "五联单-账户明细",
+            "五联单-转账明细",
+            "五联单-转账分析",
+            "五联单-账户透视",
+            "五联单-对手透视",
+            "五联单-IP地址分析",
+            "五联单-收货地址分析"
+    };
+    String[] order3 = {"财付通-开户信息", "财付通-订单明细", "财付通-交易明细", "财付通-账户透视", "财付通-对手透视", "财付通-手机明细"};
+    @Autowired
+    private GridFsTemplate gridFsTemplate;
+    @Autowired
+    private MongoTemplate mongoTemplate;
+    @Autowired
+    private GridFsOperations fsOperations;
+    @Autowired
+    @Lazy
+    private FileService fileService;
 
-  public FileService() {}
-
-  public Mapped storeUploaded(
-      InputStream inputStream,String encoding, String filename, String contentType, String modelId) {
-    ObjectId objectId;
-    objectId = this.gridFsTemplate.store(inputStream, filename, contentType);
-
-    Query query =
-            new Query(
-                    Criteria.where("modelId")
-                            .is(modelId)
-                            .and("filename")
-                            .is(filename));
-    List<LogicalFile> existsFileList = this.mongoTemplate.find(query, LogicalFile.class);
-    if(existsFileList!=null && existsFileList.size()>0){
-      return Mapped.OK();
+    public FileService() {
     }
-    LogicalFile logicalFile = new LogicalFile();
-    logicalFile.setFilename(filename);
-    logicalFile.setUploadDate(DateUtil.now());
-    logicalFile.setModelId(modelId);
-    logicalFile.setGridId(objectId);
-    logicalFile.setGenerated(Boolean.FALSE);
-    logicalFile.setStatus("LOADING");
-    logicalFile = this.mongoTemplate.insert(logicalFile);
-    this.readFile(logicalFile,encoding);
-    return Mapped.OK();
-  }
-
-  @Async
-  public void readFile(LogicalFile uploaded,String encoding) {
-    GridFSFile fsFile =
-        this.fsOperations.findOne(new Query(Criteria.where("_id").is(uploaded.getGridId())));
-    if (fsFile == null) {
-      this.modifyStatus(uploaded.getId(), "ERROR");
-    } else {
-      GridFsResource resource = this.fsOperations.getResource(fsFile);
-      String filename = resource.getFilename();
-      if (StringUtils.isBlank(filename)) {
-        this.modifyStatus(uploaded.getId(), "ERROR");
-      } else {
-        try {
-          InputStream inputStream = resource.getInputStream();
-          MongoExcelService mongoExcelService;
-          if (filename.endsWith(".xlsx")) {
-            mongoExcelService = new MongoExcelService(this.mongoTemplate, uploaded.getId());
-            ExcelXlsxReader reader = new ExcelXlsxReader(mongoExcelService);
-            reader.processStream(inputStream);
-            mongoExcelService.insertRest();
-          } else if (filename.endsWith(".xls")) {
-            mongoExcelService = new MongoExcelService(this.mongoTemplate, uploaded.getId());
-            ExcelXlsReader reader = new ExcelXlsReader(mongoExcelService);
-            reader.processStream(inputStream);
-            mongoExcelService.insertRest();
-          } else if (filename.endsWith(".csv")) {
-            mongoExcelService = new MongoExcelService(this.mongoTemplate, uploaded.getId());
-            log.info("encoding2:{}",encoding);
-            CSVParser csvParser = null;
-            if(encoding.equalsIgnoreCase("UTF-8")){
-              csvParser =  CSVFormat.EXCEL.parse(new InputStreamReader(inputStream,"utf-8"));
-            }else{
-              csvParser =  CSVFormat.EXCEL.parse(new InputStreamReader(inputStream));
-            }
-            int curRow = 0;
-            int curRowWxFile = 0;
-            boolean wxFile = false;
-            String nickname = ""; // 微信手机账单文件  添加一列 微信昵称  需要单独处理
-            for (CSVRecord record : csvParser) {
-              log.info("record,{}",record);
-              List<String> strings = record.toList();
-              List<String> strings2 = new ArrayList<>();
-              strings2.addAll(strings);
-              /*************      此处内容一下均为处理微信账单表头             ****************/
-              String join = String.join("", strings);
-              join = PublicStatic.removeAllSpecial(join);
-              if (join.equals("微信支付账单明细")) {
-                wxFile = true;
-              }
-              if (wxFile && curRowWxFile < 16) {
-                if (curRowWxFile == 1) {
-                  String temp = strings.get(0);
-                  if (StringUtils.isNotBlank(temp)) {
-                    nickname = temp.replace("微信昵称：[", "").replace("]", "");
-                  }
-                }
-                curRowWxFile++;
-                continue;
-              }
-              if (wxFile && curRowWxFile > 16) {
-                strings2.add(0, nickname);
-                curRowWxFile++;
-              }
-              if (wxFile && curRowWxFile == 16) {
-                strings2.add(0, "微信昵称");
-                curRowWxFile++;
-              }
 
-              /*************      此处内容以上均为处理微信账单表头             ****************/
-              mongoExcelService.getRows(
-                  0,
-                  curRow,
-                  strings2.stream()
-                      .map(PublicStatic::removeAllSpecial)
-                      .collect(Collectors.toList()));
-              curRow++;
-            }
-            mongoExcelService.insertRest();
-          } else if (filename.endsWith(".txt")) {
-            mongoExcelService = new MongoExcelService(this.mongoTemplate, uploaded.getId());
-            List<String> lines = new ArrayList<>();
-            IoUtil.readLines(new InputStreamReader(inputStream, "utf-8"), lines);
-            int curRow = 0;
-            for (String string : lines) {
-              if (string.equals("注销信息")) {
-                break;
-              }
-              List<String> values = Arrays.asList(string.split("\t"));
-              if (values.size() <= 1) {
-                continue;
-              }
-              mongoExcelService.getRows(
-                  0,
-                  curRow,
-                  values.stream().map(PublicStatic::removeAllSpecial).collect(Collectors.toList()));
-              curRow++;
-            }
-            mongoExcelService.insertRest();
-          }
-        } catch (Exception var11) {
-          this.modifyStatus(uploaded.getId(), "ERROR");
-          var11.printStackTrace();
+    public static String getCharsetName(File file) throws IOException {
+        String charsetName = "UTF-8";
+        // 获取 CodepageDetectorProxy 实例
+        CodepageDetectorProxy detector = CodepageDetectorProxy.getInstance();
+        // 添加解析器，会使用到添加的后 2 个 ext 里的 jar 包
+        detector.add(new ParsingDetector(false));
+        detector.add(JChardetFacade.getInstance());
+        detector.add(ASCIIDetector.getInstance());
+        detector.add(UnicodeDetector.getInstance());
+        Charset charset = detector.detectCodepage(file.toURI().toURL());
+        if (charset != null) {
+            charsetName = charset.name();
         }
-
-        this.modifyStatus(uploaded.getId(), "FINISHED");
-      }
+        return charsetName;
     }
-  }
 
-  private void modifyStatus(String id, String status) {
-    Query query = new Query();
-    query.addCriteria(Criteria.where("_id").is(id));
-    Update update = new Update();
-    update.set("status", status);
-    this.mongoTemplate.updateFirst(query, update, "logical_file");
-  }
+    public static void main(String[] args) throws IOException {
+        File file =
+                new File(
+                        "C:\\Users\\Administrator\\Desktop\\楼\\支付宝五联单数据格式\\20120303组织卖淫案_OR222713D2400411_注册信息_20210913100124.csv");
+        String charsetName = getCharsetName(file);
+        CSVParser csvParser =
+                CSVFormat.EXCEL.parse(new InputStreamReader(new FileInputStream(file), charsetName));
+        for (CSVRecord strings : csvParser) {
+            System.out.println(strings);
+        }
 
-  public Mapped listGroupUploaded(FileListInput fileListInput) {
-    String modelId = fileListInput.getModelId();
-    if (modelId == null) {
-      return Mapped.ERROR("没有选择任何分组");
-    } else {
-      Query query =
-          new Query(Criteria.where("modelId").is(modelId).and("generated").is(Boolean.TRUE));
-      query.with(Sort.by(Order.desc("uploadDate")));
-      List<LogicalFile> resultList = this.mongoTemplate.find(query, LogicalFile.class);
-      // 将文件 按照  反诈  财付通 五联单  进行分类
-      List<GroupLogincalFile> groupLogincalFiles = new ArrayList<>();
-      groupLogicalFile("反诈治安数据分析", new String[] {"反诈", "治安"}, resultList, groupLogincalFiles);
-      groupLogicalFile("支付宝数据分析", new String[] {"五联单"}, resultList, groupLogincalFiles);
-      groupLogicalFile("财付通数据分析", new String[] {"财付通"}, resultList, groupLogincalFiles);
-      Mapped res = Mapped.OK();
-      res.put("code", 200);
-      res.put("result", groupLogincalFiles);
-      return res;
+        file = new File("C:\\Users\\Administrator\\Desktop\\楼\\微信支付账单(2).csv");
+        charsetName = getCharsetName(file);
+        csvParser =
+                CSVFormat.EXCEL.parse(new InputStreamReader(new FileInputStream(file), charsetName));
+        for (CSVRecord strings : csvParser) {
+            System.out.println(strings);
+        }
     }
-  }
 
-  public void groupLogicalFile(
-      String groupName,
-      String[] group,
-      List<LogicalFile> logicalFileList,
-      List<GroupLogincalFile> groupLogincalFiles) {
-    List<LogicalFile> children = new ArrayList<>();
-    for (String s : group) {
-      children.addAll(
-          logicalFileList.stream()
-              .filter(t -> t.getFilename().startsWith(s))
-              .collect(Collectors.toList()));
-    }
-    if (children != null && children.size() > 0) {
-      GroupLogincalFile groupLogincalFile = new GroupLogincalFile();
-      groupLogincalFile.setGroupName(groupName);
-      groupLogincalFile.setId(IdUtil.randomUUID());
-      groupLogincalFile.setChildren(orderGroupLogicalFile(groupName, children));
-      groupLogincalFiles.add(groupLogincalFile);
+    public Mapped storeUploaded(
+            InputStream inputStream, String encoding, String filename, String contentType, String modelId) {
+        ObjectId objectId;
+        objectId = this.gridFsTemplate.store(inputStream, filename, contentType);
+        Query query =
+                new Query(
+                        Criteria.where("modelId")
+                                .is(modelId)
+                                .and("filename")
+                                .is(filename));
+        List<LogicalFile> existsFileList = this.mongoTemplate.find(query, LogicalFile.class);
+        if (existsFileList != null && existsFileList.size() > 0) {
+            return Mapped.OK();
+        }
+        LogicalFile logicalFile = new LogicalFile();
+        logicalFile.setFilename(filename);
+        logicalFile.setUploadDate(DateUtil.now());
+        logicalFile.setModelId(modelId);
+        logicalFile.setGridId(objectId);
+        logicalFile.setGenerated(Boolean.FALSE);
+        logicalFile.setStatus("LOADING");
+        logicalFile = this.mongoTemplate.insert(logicalFile);
+        this.readFile(logicalFile, encoding);
+        return Mapped.OK();
     }
-  }
 
-  String[] order1 = {"反诈数据-订单明细", "治安数据-订单明细", "反诈治安-交易明细", "反诈治安-账户透视", "反诈治安-对手透视"};
-  String[] order2 = {
-    "五联单-注册信息",
-    "五联单-登录日志",
-    "五联单-交易记录",
-    "五联单-账户明细",
-    "五联单-转账明细",
-    "五联单-转账分析",
-    "五联单-账户透视",
-    "五联单-对手透视",
-    "五联单-IP地址分析",
-    "五联单-收货地址分析"
-  };
-  String[] order3 = {"财付通-开户信息", "财付通-订单明细", "财付通-交易明细", "财付通-账户透视", "财付通-对手透视", "财付通-手机明细"};
+    @Async
+    public void readFile(LogicalFile uploaded, String encoding) {
+        GridFSFile fsFile =
+                this.fsOperations.findOne(new Query(Criteria.where("_id").is(uploaded.getGridId())));
+        if (fsFile == null) {
+            this.modifyStatus(uploaded.getId(), "ERROR");
+        } else {
+            GridFsResource resource = this.fsOperations.getResource(fsFile);
+            String filename = resource.getFilename();
+            if (StringUtils.isBlank(filename)) {
+                this.modifyStatus(uploaded.getId(), "ERROR");
+            } else {
+                try {
+                    InputStream inputStream = resource.getInputStream();
+                    MongoExcelService mongoExcelService;
+                    if (filename.endsWith(".xlsx")) {
+                        mongoExcelService = new MongoExcelService(this.mongoTemplate, uploaded.getId());
+                        ExcelXlsxReader reader = new ExcelXlsxReader(mongoExcelService);
+                        reader.processStream(inputStream);
+                        mongoExcelService.insertRest();
+                    } else if (filename.endsWith(".xls")) {
+                        mongoExcelService = new MongoExcelService(this.mongoTemplate, uploaded.getId());
+                        ExcelXlsReader reader = new ExcelXlsReader(mongoExcelService);
+                        reader.processStream(inputStream);
+                        mongoExcelService.insertRest();
+                    } else if (filename.endsWith(".csv")) {
+                        mongoExcelService = new MongoExcelService(this.mongoTemplate, uploaded.getId());
+                        log.info("encoding2:{}", encoding);
+                        CSVParser csvParser = null;
+                        if (encoding.equalsIgnoreCase("UTF-8")) {
+                            csvParser = CSVFormat.EXCEL.parse(new InputStreamReader(inputStream, "utf-8"));
+                        } else {
+                            csvParser = CSVFormat.EXCEL.parse(new InputStreamReader(inputStream));
+                        }
+                        int curRow = 0;
+                        int curRowWxFile = 0;
+                        boolean wxFile = false;
+                        String nickname = ""; // 微信手机账单文件  添加一列 微信昵称  需要单独处理
+                        for (CSVRecord record : csvParser) {
+                            log.info("record,{}", record);
+                            List<String> strings = record.toList();
+                            List<String> strings2 = new ArrayList<>();
+                            strings2.addAll(strings);
+                            /*************      此处内容一下均为处理微信账单表头             ****************/
+                            String join = String.join("", strings);
+                            join = PublicStatic.removeAllSpecial(join);
+                            if (join.equals("微信支付账单明细")) {
+                                wxFile = true;
+                            }
+                            if (wxFile && curRowWxFile < 16) {
+                                if (curRowWxFile == 1) {
+                                    String temp = strings.get(0);
+                                    if (StringUtils.isNotBlank(temp)) {
+                                        nickname = temp.replace("微信昵称：[", "").replace("]", "");
+                                    }
+                                }
+                                curRowWxFile++;
+                                continue;
+                            }
+                            if (wxFile && curRowWxFile > 16) {
+                                strings2.add(0, nickname);
+                                curRowWxFile++;
+                            }
+                            if (wxFile && curRowWxFile == 16) {
+                                strings2.add(0, "微信昵称");
+                                curRowWxFile++;
+                            }
 
-  public List<LogicalFile> orderGroupLogicalFile(String groupName, List<LogicalFile> list) {
+                            /*************      此处内容以上均为处理微信账单表头             ****************/
+                            mongoExcelService.getRows(
+                                    0,
+                                    curRow,
+                                    strings2.stream()
+                                            .map(PublicStatic::removeAllSpecial)
+                                            .collect(Collectors.toList()));
+                            curRow++;
+                        }
+                        mongoExcelService.insertRest();
+                    } else if (filename.endsWith(".txt")) {
+                        mongoExcelService = new MongoExcelService(this.mongoTemplate, uploaded.getId());
+                        List<String> lines = new ArrayList<>();
+                        IoUtil.readLines(new InputStreamReader(inputStream, "utf-8"), lines);
+                        int curRow = 0;
+                        for (String string : lines) {
+                            if (string.equals("注销信息")) {
+                                break;
+                            }
+                            List<String> values = Arrays.asList(string.split("\t"));
+                            if (values.size() <= 1) {
+                                continue;
+                            }
+                            mongoExcelService.getRows(
+                                    0,
+                                    curRow,
+                                    values.stream().map(PublicStatic::removeAllSpecial).collect(Collectors.toList()));
+                            curRow++;
+                        }
+                        mongoExcelService.insertRest();
+                    }
+                } catch (Exception var11) {
+                    this.modifyStatus(uploaded.getId(), "ERROR");
+                    var11.printStackTrace();
+                }
 
-    if (groupName.equals("反诈治安数据分析")) {
-      return orderGroupLogicalFileChild(order1, list);
-    }
-    if (groupName.equals("支付宝数据分析")) {
-      return orderGroupLogicalFileChild(order2, list);
-    }
-    if (groupName.equals("财付通数据分析")) {
-      return orderGroupLogicalFileChild(order3, list);
+                this.modifyStatus(uploaded.getId(), "FINISHED");
+            }
+        }
     }
-    return null;
-  }
 
-  public List<LogicalFile> orderGroupLogicalFileChild(String[] orders, List<LogicalFile> list) {
-    List<LogicalFile> child = new ArrayList<>();
-    for (String s : orders) {
+    private void modifyStatus(String id, String status) {
+        Query query = new Query();
+        query.addCriteria(Criteria.where("_id").is(id));
+        Update update = new Update();
+        update.set("status", status);
+        this.mongoTemplate.updateFirst(query, update, "logical_file");
+    }
 
-      for (LogicalFile logicalFile : list) {
-        if (logicalFile.getFilename().equals(s)) {
-          child.add(logicalFile);
-          break;
+    public Mapped listGroupUploaded(FileListInput fileListInput) {
+        String modelId = fileListInput.getModelId();
+        if (modelId == null) {
+            return Mapped.ERROR("没有选择任何分组");
+        } else {
+            Query query =
+                    new Query(Criteria.where("modelId").is(modelId).and("generated").is(Boolean.TRUE));
+            query.with(Sort.by(Order.desc("uploadDate")));
+            List<LogicalFile> resultList = this.mongoTemplate.find(query, LogicalFile.class);
+            // 将文件 按照  反诈  财付通 五联单  进行分类
+            List<GroupLogincalFile> groupLogincalFiles = new ArrayList<>();
+            groupLogicalFile("反诈治安数据分析", new String[]{"反诈", "治安"}, resultList, groupLogincalFiles);
+            groupLogicalFile("支付宝数据分析", new String[]{"五联单"}, resultList, groupLogincalFiles);
+            groupLogicalFile("财付通数据分析", new String[]{"财付通"}, resultList, groupLogincalFiles);
+            Mapped res = Mapped.OK();
+            res.put("code", 200);
+            res.put("result", groupLogincalFiles);
+            return res;
         }
-      }
     }
-    return child;
-  }
 
-  public Mapped listUploaded(FileListInput fileListInput) {
-    String modelId = fileListInput.getModelId();
-    if (modelId == null) {
-      return Mapped.ERROR("没有选择任何分组");
-    } else {
-      String generated = fileListInput.getGenerated();
-      Future<Long> longFuture = this.fileService.countTotalByModelId(modelId, generated);
-      Future<String> modelNameFuture = this.fileService.retrieveModelname(modelId);
-      int page = fileListInput.getPage();
-      int pageSize = fileListInput.getPageSize();
-      int skip = (page - 1) * pageSize;
-      Criteria criteria = Criteria.where("modelId").is(modelId);
-      if ("true".equals(generated)) {
-        criteria.and("generated").is(Boolean.TRUE);
-      } else if ("false".equals(generated)) {
-        criteria.and("generated").is(Boolean.FALSE);
-      }
-      Query query = new Query(criteria);
-      query.with(Sort.by(Order.desc("uploadDate")));
-      List<LogicalFile> resultList =
-          this.mongoTemplate.find(query.skip(skip).limit(pageSize), LogicalFile.class);
-      Long total;
-      try {
-        total = longFuture.get();
-      } catch (ExecutionException | InterruptedException var17) {
-        return Mapped.ERROR("查询出错");
-      }
-      String modelName;
-      try {
-        modelName = modelNameFuture.get();
-      } catch (ExecutionException | InterruptedException var17) {
-        return Mapped.ERROR("查询出错");
-      }
-      int i = skip;
-      for (LogicalFile file : resultList) {
-        ++i;
-        file.setCount(i);
-        file.setModelName(modelName);
-      }
-      Mapped res = Mapped.OK();
-      res.put("code", 200);
-      res.put("result", resultList);
-      res.put("total", total);
-      res.put("page", page);
-      res.put("pageSize", pageSize);
-      return res;
+    public void groupLogicalFile(
+            String groupName,
+            String[] group,
+            List<LogicalFile> logicalFileList,
+            List<GroupLogincalFile> groupLogincalFiles) {
+        List<LogicalFile> children = new ArrayList<>();
+        for (String s : group) {
+            children.addAll(
+                    logicalFileList.stream()
+                            .filter(t -> t.getFilename().startsWith(s))
+                            .collect(Collectors.toList()));
+        }
+        if (children != null && children.size() > 0) {
+            GroupLogincalFile groupLogincalFile = new GroupLogincalFile();
+            groupLogincalFile.setGroupName(groupName);
+            groupLogincalFile.setId(IdUtil.randomUUID());
+            groupLogincalFile.setChildren(orderGroupLogicalFile(groupName, children));
+            groupLogincalFiles.add(groupLogincalFile);
+        }
     }
-  }
 
-  public Mapped deleteById(IdInput idInput) {
-    String id = idInput.getId();
-    Query query = new Query(Criteria.where("id").is(id));
-    LogicalFile file = this.mongoTemplate.findOne(query, LogicalFile.class);
-    if (file != null && file.getGridId() != null) {
-      String physicalId = file.getGridId().toHexString();
-      this.fileService.removePhysicalFile(physicalId);
+    public List<LogicalFile> orderGroupLogicalFile(String groupName, List<LogicalFile> list) {
+
+        if (groupName.equals("反诈治安数据分析")) {
+            return orderGroupLogicalFileChild(order1, list);
+        }
+        if (groupName.equals("支付宝数据分析")) {
+            return orderGroupLogicalFileChild(order2, list);
+        }
+        if (groupName.equals("财付通数据分析")) {
+            return orderGroupLogicalFileChild(order3, list);
+        }
+        return null;
     }
 
-    this.fileService.removeDataByFileId(id);
-    this.fileService.removeFieldsByFileId(id);
-    DeleteResult result = this.mongoTemplate.remove(query, LogicalFile.class);
-    return result.getDeletedCount() > 0L ? Mapped.OK() : Mapped.ERROR("删除失败");
-  }
+    public List<LogicalFile> orderGroupLogicalFileChild(String[] orders, List<LogicalFile> list) {
+        List<LogicalFile> child = new ArrayList<>();
+        for (String s : orders) {
+
+            for (LogicalFile logicalFile : list) {
+                if (logicalFile.getFilename().equals(s)) {
+                    child.add(logicalFile);
+                    break;
+                }
+            }
+        }
+        return child;
+    }
 
-  @Async
-  Future<Long> countTotalByModelId(String modelId, String generated) {
-    Criteria criteria = Criteria.where("modelId").is(modelId);
-    if ("true".equals(generated)) {
-      criteria.and("generated").is(Boolean.TRUE);
-    } else if ("false".equals(generated)) {
-      criteria.and("generated").is(Boolean.FALSE);
+    public Mapped listUploaded(FileListInput fileListInput) {
+        String modelId = fileListInput.getModelId();
+        if (modelId == null) {
+            return Mapped.ERROR("没有选择任何分组");
+        } else {
+            String generated = fileListInput.getGenerated();
+            Future<Long> longFuture = this.fileService.countTotalByModelId(modelId, generated);
+            Future<String> modelNameFuture = this.fileService.retrieveModelname(modelId);
+            int page = fileListInput.getPage();
+            int pageSize = fileListInput.getPageSize();
+            int skip = (page - 1) * pageSize;
+            Criteria criteria = Criteria.where("modelId").is(modelId);
+            if ("true".equals(generated)) {
+                criteria.and("generated").is(Boolean.TRUE);
+            } else if ("false".equals(generated)) {
+                criteria.and("generated").is(Boolean.FALSE);
+            }
+            Query query = new Query(criteria);
+            query.with(Sort.by(Order.desc("uploadDate")));
+            List<LogicalFile> resultList =
+                    this.mongoTemplate.find(query.skip(skip).limit(pageSize), LogicalFile.class);
+            Long total;
+            try {
+                total = longFuture.get();
+            } catch (ExecutionException | InterruptedException var17) {
+                return Mapped.ERROR("查询出错");
+            }
+            String modelName;
+            try {
+                modelName = modelNameFuture.get();
+            } catch (ExecutionException | InterruptedException var17) {
+                return Mapped.ERROR("查询出错");
+            }
+            int i = skip;
+            for (LogicalFile file : resultList) {
+                ++i;
+                file.setCount(i);
+                file.setModelName(modelName);
+            }
+            Mapped res = Mapped.OK();
+            res.put("code", 200);
+            res.put("result", resultList);
+            res.put("total", total);
+            res.put("page", page);
+            res.put("pageSize", pageSize);
+            return res;
+        }
     }
 
-    Long total = this.mongoTemplate.count(new Query(criteria), LogicalFile.class);
-    return new AsyncResult(total);
-  }
+    public Mapped deleteById(IdInput idInput) {
+        String id = idInput.getId();
+        Query query = new Query(Criteria.where("id").is(id));
+        LogicalFile file = this.mongoTemplate.findOne(query, LogicalFile.class);
+        if (file != null && file.getGridId() != null) {
+            String physicalId = file.getGridId().toHexString();
+            this.fileService.removePhysicalFile(physicalId);
+        }
 
-  @Async
-  public void removePhysicalFile(String id) {
-    this.fsOperations.delete(new Query(Criteria.where("_id").is(id)));
-  }
+        this.fileService.removeDataByFileId(id);
+        this.fileService.removeFieldsByFileId(id);
+        DeleteResult result = this.mongoTemplate.remove(query, LogicalFile.class);
+        return result.getDeletedCount() > 0L ? Mapped.OK() : Mapped.ERROR("删除失败");
+    }
 
-  @Async
-  public void removeLogicalFile(String fileId) {
-    this.mongoTemplate.remove(new Query(Criteria.where("id").is(fileId)), LogicalFile.class);
-  }
+    @Async
+    Future<Long> countTotalByModelId(String modelId, String generated) {
+        Criteria criteria = Criteria.where("modelId").is(modelId);
+        if ("true".equals(generated)) {
+            criteria.and("generated").is(Boolean.TRUE);
+        } else if ("false".equals(generated)) {
+            criteria.and("generated").is(Boolean.FALSE);
+        }
 
-  @Async
-  public void removeDataByFileId(String fileId) {
-    this.mongoTemplate.remove(new Query(Criteria.where("fileId").is(fileId)), DataMap.class);
-  }
+        Long total = this.mongoTemplate.count(new Query(criteria), LogicalFile.class);
+        return new AsyncResult(total);
+    }
 
-  @Async
-  public void removeFieldsByFileId(String fileId) {
-    this.mongoTemplate.remove(new Query(Criteria.where("fileId").is(fileId)), Fields.class);
-  }
+    @Async
+    public void removePhysicalFile(String id) {
+        this.fsOperations.delete(new Query(Criteria.where("_id").is(id)));
+    }
 
-  @Async
-  Future<String> retrieveModelname(String modelId) {
-    Query query = new Query(Criteria.where("id").is(modelId));
-    query.fields().include("modelName");
-    Model model = this.mongoTemplate.findOne(query, Model.class, "model");
-    return new AsyncResult(model != null ? model.getModelName() : "");
-  }
+    @Async
+    public void removeLogicalFile(String fileId) {
+        this.mongoTemplate.remove(new Query(Criteria.where("id").is(fileId)), LogicalFile.class);
+    }
 
-  public static String getCharsetName(File file) throws IOException {
-    String charsetName = "UTF-8";
-    // 获取 CodepageDetectorProxy 实例
-    CodepageDetectorProxy detector = CodepageDetectorProxy.getInstance();
-    // 添加解析器，会使用到添加的后 2 个 ext 里的 jar 包
-    detector.add(new ParsingDetector(false));
-    detector.add(JChardetFacade.getInstance());
-    detector.add(ASCIIDetector.getInstance());
-    detector.add(UnicodeDetector.getInstance());
-    Charset charset = detector.detectCodepage(file.toURI().toURL());
-    if (charset != null) {
-      charsetName = charset.name();
+    @Async
+    public void removeDataByFileId(String fileId) {
+        this.mongoTemplate.remove(new Query(Criteria.where("fileId").is(fileId)), DataMap.class);
     }
-    return charsetName;
-  }
 
-  public static void main(String[] args) throws IOException {
-    File file =
-        new File(
-            "C:\\Users\\Administrator\\Desktop\\楼\\支付宝五联单数据格式\\20120303组织卖淫案_OR222713D2400411_注册信息_20210913100124.csv");
-    String charsetName = getCharsetName(file);
-    CSVParser csvParser =
-        CSVFormat.EXCEL.parse(new InputStreamReader(new FileInputStream(file), charsetName));
-    for (CSVRecord strings : csvParser) {
-      System.out.println(strings);
+    @Async
+    public void removeFieldsByFileId(String fileId) {
+        this.mongoTemplate.remove(new Query(Criteria.where("fileId").is(fileId)), Fields.class);
     }
 
-    file = new File("C:\\Users\\Administrator\\Desktop\\楼\\微信支付账单(2).csv");
-    charsetName = getCharsetName(file);
-    csvParser =
-        CSVFormat.EXCEL.parse(new InputStreamReader(new FileInputStream(file), charsetName));
-    for (CSVRecord strings : csvParser) {
-      System.out.println(strings);
+    @Async
+    Future<String> retrieveModelname(String modelId) {
+        Query query = new Query(Criteria.where("id").is(modelId));
+        query.fields().include("modelName");
+        Model model = this.mongoTemplate.findOne(query, Model.class, "model");
+        return new AsyncResult(model != null ? model.getModelName() : "");
     }
-  }
 }

src/main/java/ieven/server/webapp/domain/tenpay/TenpayHeaders.java

@@ -6,86 +6,78 @@ import java.util.List;
 import java.util.Map;
 
 public class TenpayHeaders {
-  private static final Map<Integer, List<String>> mappedHeaders = new HashMap<>();
-  private static final Map<Integer, String> mappedNames = new HashMap<>();
+    private static final Map<Integer, List<String>> mappedHeaders = new HashMap<>();
+    private static final Map<Integer, String> mappedNames = new HashMap<>();
 
-  public TenpayHeaders() {}
+    static {
+        mappedHeaders.put(1, Arrays.asList(
+                "用户ID", "交易单号", "借贷类型", "交易类型", "交易金额(分)", "账户余额(分)", "交易时间", "银行类型", "交易说明", "商户名称",
+                "发送方", "发送金额(分)", "接收方", "接收时间", "接收金额(分)", "银行卡号", "网银联单号1", "网银联单号2"));
 
-  public static List<String> getHeaders(int type) {
-    return mappedHeaders.get(type);
-  }
+        mappedHeaders.put(2, Arrays.asList(
+                "用户ID",
+                "交易单号",
+                "交易主体账号",
+                "交易主体银行卡号",
+                "交易主体银行名称",
+                "交易主体户名",
+                "借贷类型",
+                "交易类型",
+                "交易时间",
+                "交易金额(元)",
+                "账户余额(元)",
+                "银行类型",
+                "交易说明",
+                "交易对手账号",
+                "交易对手银行卡号",
+                "交易对手银行名称",
+                "交易对手户名",
+                "网银联单号1",
+                "网银联单号2"));
+        mappedHeaders.put(3, Arrays.asList("账户状态", "账号", "注册姓名", "注册时间", "注册身份证号", "绑定手机", "绑定状态", "开户行信息", "银行账号"));
+        mappedHeaders.put(4, Arrays.asList("微信昵称", "交易时间", "交易类型", "交易对方", "商品", "收/支", "金额(元)", "支付方式", "当前状态", "交易单号", "商户单号", "备注"));
+        mappedHeaders.put(5, Arrays.asList(
+                "用户ID",
+                "交易单号",
+                "借贷类型",
+                "交易业务类型",
+                "交易用途类型",
+                "交易时间",
+                "交易金额(分)",
+                "账户余额(分)",
+                "用户银行卡号",
+                "用户侧网银联单号",
+                "网联/银联",
+                "第三方账户名称",
+                "对手方ID",
+                "对手方银行卡号",
+                "对手侧网银联单号",
+                "网联/银联",
+                "第三方账户名称",
+                "对手方接收时间",
+                "对手方接收金额(分)",
+                "备注1",
+                "备注2"));
 
-  public static String getGeneratedName(int type) {
-    return mappedNames.get(type);
-  }
+        mappedHeaders.put(6, Arrays.asList(
+                "用户ID", "交易单号", "借贷类型", "交易类型", "交易金额(分)", "账户余额(分)", "交易时间", "银行类型", "交易说明", "商户名称", "发送方", "发送金额(分)", "接收方", "接收时间", "接收金额(分)", "银行卡号", "对手银行卡号", "网银联单号1", "网银联单号2"));
+        mappedHeaders.put(7, Arrays.asList(
+                "用户ID", "交易单号", "大单号", "用户侧账号名称", "借贷类型", "交易业务类型", "交易用途类型", "交易时间", "交易金额(分)", "账户余额(分)", "用户银行卡号", "用户侧银行名称", "用户侧网银联单号", "网联/银联", "第三方账户名称", "对手方ID", "对手侧账户名称", "对手方银行卡号", "对手侧银行名称", "对手侧网银联单号", "网联/银联", "第三方账户名称", "对手方接收时间", "对手方接收金额(分)", "备注1"
+        ));
 
-  static {
-    mappedHeaders.put(
-        1,
-        Arrays.asList(
-            "用户ID", "交易单号", "借贷类型", "交易类型", "交易金额(分)", "账户余额(分)", "交易时间", "银行类型", "交易说明", "商户名称",
-            "发送方", "发送金额(分)", "接收方", "接收时间", "接收金额(分)", "银行卡号", "网银联单号1", "网银联单号2"));
+        mappedNames.put(1, "财付通-订单明细");
 
-    mappedHeaders.put(
-        2,
-        Arrays.asList(
-            "用户ID",
-            "交易单号",
-            "交易主体账号",
-            "交易主体银行卡号",
-            "交易主体银行名称",
-            "交易主体户名",
-            "借贷类型",
-            "交易类型",
-            "交易时间",
-            "交易金额(元)",
-            "账户余额(元)",
-            "银行类型",
-            "交易说明",
-            "交易对手账号",
-            "交易对手银行卡号",
-            "交易对手银行名称",
-            "交易对手户名",
-            "网银联单号1",
-            "网银联单号2"));
-    mappedHeaders.put(
-        3, Arrays.asList("账户状态", "账号", "注册姓名", "注册时间", "注册身份证号", "绑定手机", "绑定状态", "开户行信息", "银行账号"));
-    mappedHeaders.put(
-        4,
-        Arrays.asList(
-            "微信昵称", "交易时间", "交易类型", "交易对方", "商品", "收/支", "金额(元)", "支付方式", "当前状态", "交易单号", "商户单号",
-            "备注"));
-    mappedHeaders.put(
-        5,
-        Arrays.asList(
-            "用户ID",
-            "交易单号",
-            "借贷类型",
-            "交易业务类型",
-            "交易用途类型",
-            "交易时间",
-            "交易金额(分)",
-            "账户余额(分)",
-            "用户银行卡号",
-            "用户侧网银联单号",
-            "网联/银联",
-            "第三方账户名称",
-            "对手方ID",
-            "对手方银行卡号",
-            "对手侧网银联单号",
-            "网联/银联",
-            "第三方账户名称",
-            "对手方接收时间",
-            "对手方接收金额(分)",
-            "备注1",
-            "备注2"));
 
-    mappedHeaders.put(
-        6,
-        Arrays.asList(
-            "用户ID", "交易单号", "借贷类型", "交易类型", "交易金额(分)", "账户余额(分)", "交易时间", "银行类型", "交易说明", "商户名称",
-            "发送方", "发送金额(分)", "接收方", "接收时间", "接收金额(分)", "银行卡号", "对手银行卡号", "网银联单号1", "网银联单号2"));
+    }
 
-    mappedNames.put(1, "财付通-订单明细");
-  }
+    public TenpayHeaders() {
+    }
+
+    public static List<String> getHeaders(int type) {
+        return mappedHeaders.get(type);
+    }
+
+    public static String getGeneratedName(int type) {
+        return mappedNames.get(type);
+    }
 }

src/main/java/ieven/server/webapp/service/FieldsService.java

@@ -65,6 +65,8 @@ public class FieldsService extends Ops {
     this.updateLogicalFile(fields, "财付通-订单明细");
     fields = matchHeaders(originFields, TenpayHeaders.getHeaders(2));
     this.updateLogicalFile(fields, "财付通-订单明细");
+    fields = matchHeaders(originFields, TenpayHeaders.getHeaders(7));
+    this.updateLogicalFile(fields, "财付通-订单明细");
     // 财付通-手机明细
     fields = matchHeaders(originFields, TenpayHeaders.getHeaders(4));
     this.updateLogicalFile(fields, "财付通-手机明细");

src/main/java/ieven/server/webapp/service/Ops.java

@@ -26,254 +26,271 @@ import java.util.regex.Pattern;
 
 public class Ops {
 
-  public Ops() {}
-
-  protected void replaceKey(DataMap dataMap, String oldKey, String newKey) {
-    Object valueOld = dataMap.get(oldKey);
-    dataMap.remove(oldKey);
-    dataMap.put(newKey, valueOld);
-  }
+    public Ops() {
+    }
 
-  protected MongoCursor<Document> getCursor(String inputFileId, MongoTemplate mongoTemplate) {
-    Query query = new Query(Criteria.where("fileId").is(inputFileId));
-    // 查询游标
-    return mongoTemplate
-        .getCollection("data")
-        .find(query.getQueryObject())
-        .batchSize(1000)
-        .noCursorTimeout(true)
-        .cursor();
-  }
+    public static Fields createFile(int type, String modelId) {
+        List<String> headerNames = AlipayHeaders.getHeaders(type);
+        headerNames.replaceAll(String::toUpperCase);
+        String filename = AlipayHeaders.getGeneratedName(type);
+        MongoTemplate mongoTemplate = SpringContextUtil.getBean(MongoTemplate.class);
+        FileService fileService = SpringContextUtil.getBean(FileService.class);
+        String regex = "^.*" + filename.trim() + ".*$";
+        Pattern pattern = Pattern.compile(regex, 2);
+        Query query =
+                new Query(
+                        Criteria.where("modelId")
+                                .is(modelId)
+                                .and("filename")
+                                .regex(pattern)
+                                .and("generated")
+                                .is(Boolean.TRUE));
+        List<LogicalFile> oldFiles = mongoTemplate.find(query, LogicalFile.class);
 
-  protected List<Fields> checkFields(List<Fields> fieldsList, int requireType) {
-    List<String> requireHeaders = AlipayHeaders.getHeaders(requireType);
-    List<Fields> checked = new ArrayList<>();
-    for (Fields fields : fieldsList) {
-      List<String> headers = new ArrayList<>(fields.getFieldsReverse().keySet());
-      // 字段完全一样就说明是需要查找的表
-      if (headers.containsAll(requireHeaders) && requireHeaders.containsAll(headers)) {
-        checked.add(fields);
-      }
+        for (LogicalFile old : oldFiles) {
+            IdInput idInput = new IdInput();
+            idInput.setId(old.getId());
+            fileService.deleteById(idInput);
+        }
+        LogicalFile logicalFile = new LogicalFile();
+        logicalFile.setFilename(filename);
+        logicalFile.setStatus("LOADING");
+        String current = DataUtils.currentDate();
+        logicalFile.setUploadDate(current);
+        logicalFile.setModelId(modelId);
+        logicalFile.setGenerated(Boolean.TRUE);
+        logicalFile = mongoTemplate.insert(logicalFile);
+        Fields fields = new Fields();
+        fields.setFileId(logicalFile.getId());
+        LinkedHashMap<String, String> header = new LinkedHashMap<>();
+        LinkedHashMap<String, String> headerReversed = new LinkedHashMap<>();
+        for (String headerName : headerNames) {
+            header.put(headerName, headerName);
+            headerReversed.put(headerName, headerName);
+        }
+        fields.setFields(header);
+        fields.setFieldsReverse(headerReversed);
+        return mongoTemplate.insert(fields);
     }
-    return checked;
-  }
 
-  protected DataMap initWithOrigin(Document originData, String newFileId) {
-    DataMap newData = new DataMap();
-    // 遍历原始表,复制每一个值到dataMap并且转换成大写字母
-    for (Map.Entry<String, Object> entry : originData.entrySet()) {
-      String key = entry.getKey();
-      Object value = entry.getValue();
-      if (!"id".equals(key) && !"_id".equals(key) && !"objectId".equals(key)) {
-        String upper = key.toUpperCase();
-        newData.put(upper, value);
-      }
-    }
-    newData.put("fileId", newFileId);
-    return newData;
-  }
+    public static List<String> extract_1(String origin) {
+        List<String> stringList = new ArrayList<>();
+        stringList.add(origin.replaceAll("\\(.*\\)", ""));
+        Matcher mat = Pattern.compile("(?<=\\()[^)]+").matcher(origin);
 
-  public void saveLines(List<DataMap> documentList, MongoTemplate mongoTemplate) {
-    List<Document> manyList = new ArrayList<>();
-    for (DataMap dataMap : documentList) {
-      manyList.add(dataMap.formHash());
-    }
-    InsertManyOptions options = new InsertManyOptions();
-    options.ordered(false);
-    try {
-      mongoTemplate.getCollection("data").insertMany(manyList, options);
-    } catch (MongoBulkWriteException | DuplicateKeyException ignored) {
+        while (mat.find()) {
+            stringList.add(mat.group());
+        }
+
+        return stringList;
     }
-  }
 
-  public void saveLines(
-      Collection<DataMap> documentList, String sortField, MongoTemplate mongoTemplate) {
-    List<Document> manyList = new ArrayList<>();
-    for (DataMap dataMap : documentList) {
-      manyList.add(dataMap.formHash());
+    public static String getString(Document doc, Fields fields, String chineseHead) {
+        LinkedHashMap<String, String> inputReversed = fields.getFieldsReverse();
+        String alias = inputReversed.get(chineseHead);
+        return StringUtils.isNotBlank(alias) ? doc.get(alias, "") : "";
     }
-    // 按照某个字段倒序排序
-    manyList.sort(
-        new Comparator<Document>() {
-          @Override
-          public int compare(Document o1, Document o2) {
-            return (Double.valueOf(String.valueOf(o2.get(sortField)))
-                .compareTo(Double.valueOf(String.valueOf(o1.get(sortField)))));
-          }
-        });
-    InsertManyOptions options = new InsertManyOptions();
-    options.ordered(false);
-    try {
-      List<Document> batchInsertList = new ArrayList<>();
-      int k = 0;
-      for (Document document : manyList) {
-        batchInsertList.add(document);
-        k++;
-        if (k >= 1000) {
-          mongoTemplate.getCollection("data").insertMany(batchInsertList, options);
-          batchInsertList = new ArrayList<>();
-          k = 0;
+
+    public static void setString(DataMap doc, String chineseHead, Fields fields, String value) {
+        LinkedHashMap<String, String> inputReversed = fields.getFieldsReverse();
+        String alias = inputReversed.get(chineseHead);
+        if (StringUtils.isNotBlank(alias)) {
+            doc.put(alias, value);
         }
-      }
-      if (batchInsertList.size() > 0) {
-        mongoTemplate.getCollection("data").insertMany(batchInsertList, options);
-      }
-    } catch (Exception exception) {
-      exception.printStackTrace();
     }
-  }
 
-  public static Fields createFile(int type, String modelId) {
-    List<String> headerNames = AlipayHeaders.getHeaders(type);
-    headerNames.replaceAll(String::toUpperCase);
-    String filename = AlipayHeaders.getGeneratedName(type);
-    MongoTemplate mongoTemplate = SpringContextUtil.getBean(MongoTemplate.class);
-    FileService fileService = SpringContextUtil.getBean(FileService.class);
-    String regex = "^.*" + filename.trim() + ".*$";
-    Pattern pattern = Pattern.compile(regex, 2);
-    Query query =
-        new Query(
-            Criteria.where("modelId")
-                .is(modelId)
-                .and("filename")
-                .regex(pattern)
-                .and("generated")
-                .is(Boolean.TRUE));
-    List<LogicalFile> oldFiles = mongoTemplate.find(query, LogicalFile.class);
-
-    for (LogicalFile old : oldFiles) {
-      IdInput idInput = new IdInput();
-      idInput.setId(old.getId());
-      fileService.deleteById(idInput);
-    }
-    LogicalFile logicalFile = new LogicalFile();
-    logicalFile.setFilename(filename);
-    logicalFile.setStatus("LOADING");
-    String current = DataUtils.currentDate();
-    logicalFile.setUploadDate(current);
-    logicalFile.setModelId(modelId);
-    logicalFile.setGenerated(Boolean.TRUE);
-    logicalFile = mongoTemplate.insert(logicalFile);
-    Fields fields = new Fields();
-    fields.setFileId(logicalFile.getId());
-    LinkedHashMap<String, String> header = new LinkedHashMap<>();
-    LinkedHashMap<String, String> headerReversed = new LinkedHashMap<>();
-    for (String headerName : headerNames) {
-      header.put(headerName, headerName);
-      headerReversed.put(headerName, headerName);
+    protected void replaceKey(DataMap dataMap, String oldKey, String newKey) {
+        Object valueOld = dataMap.get(oldKey);
+        dataMap.remove(oldKey);
+        dataMap.put(newKey, valueOld);
     }
-    fields.setFields(header);
-    fields.setFieldsReverse(headerReversed);
-    return mongoTemplate.insert(fields);
-  }
 
-  protected String createLogicAndDeleteOld(
-      String filename,
-      String modelId,
-      List<String> newHeaders,
-      MongoTemplate mongoTemplate,
-      FileService fileService) {
-    // 模糊匹配出所有的文件并删除
-    Query query =
-        new Query(
-            Criteria.where("modelId")
-                .is(modelId)
-                .and("filename")
-                .is(filename)
-                .and("generated")
-                .is(Boolean.TRUE));
-    List<LogicalFile> oldFiles = mongoTemplate.find(query, LogicalFile.class);
-    for (LogicalFile old : oldFiles) {
-      IdInput idInput = new IdInput();
-      idInput.setId(old.getId());
-      fileService.deleteById(idInput);
+    protected MongoCursor<Document> getCursor(String inputFileId, MongoTemplate mongoTemplate) {
+        Query query = new Query(Criteria.where("fileId").is(inputFileId));
+        // 查询游标
+        return mongoTemplate
+                .getCollection("data")
+                .find(query.getQueryObject())
+                .batchSize(1000)
+                .noCursorTimeout(true)
+                .cursor();
     }
-    // 先删除原来的文件再创建
-    LogicalFile logicalFile = new LogicalFile();
-    logicalFile.setFilename(filename);
-    logicalFile.setStatus(FileStatus.STATUS_FINISHED);
-    String current = DataUtils.currentDate();
-    logicalFile.setUploadDate(current);
-    logicalFile.setModelId(modelId);
-    logicalFile.setGenerated(Boolean.TRUE);
-    // 插入文件获取文件id
-    logicalFile = mongoTemplate.insert(logicalFile);
-    Fields fields = new Fields();
-    fields.setFileId(logicalFile.getId());
-    // 插入表头
-    LinkedHashMap<String, String> header = new LinkedHashMap<>();
-    LinkedHashMap<String, String> headerReversed = new LinkedHashMap<>();
-    for (String field : newHeaders) {
-      header.put(field, field);
-      headerReversed.put(field, field);
+
+    protected List<Fields> checkFields(List<Fields> fieldsList, int requireType) {
+        List<String> requireHeaders = AlipayHeaders.getHeaders(requireType);
+        List<Fields> checked = new ArrayList<>();
+        for (Fields fields : fieldsList) {
+            List<String> headers = new ArrayList<>(fields.getFieldsReverse().keySet());
+            // 字段完全一样就说明是需要查找的表
+            if (headers.containsAll(requireHeaders) && requireHeaders.containsAll(headers)) {
+                checked.add(fields);
+            }
+        }
+        return checked;
     }
-    fields.setFields(header);
-    fields.setFieldsReverse(headerReversed);
-    mongoTemplate.insert(fields);
-    return logicalFile.getId();
-  }
 
-  public static List<String> extract_1(String origin) {
-    List<String> stringList = new ArrayList<>();
-    stringList.add(origin.replaceAll("\\(.*\\)", ""));
-    Matcher mat = Pattern.compile("(?<=\\()[^)]+").matcher(origin);
+    protected DataMap initWithOrigin(Document originData, String newFileId) {
+        DataMap newData = new DataMap();
+        // 遍历原始表,复制每一个值到dataMap并且转换成大写字母
+        for (Map.Entry<String, Object> entry : originData.entrySet()) {
+            String key = entry.getKey();
+            Object value = entry.getValue();
+            if (!"id".equals(key) && !"_id".equals(key) && !"objectId".equals(key)) {
+                String upper = key.toUpperCase();
+                newData.put(upper, value);
+            }
+        }
+        newData.put("fileId", newFileId);
+        return newData;
+    }
 
-    while (mat.find()) {
-      stringList.add(mat.group());
+    public void saveLines(List<DataMap> documentList, MongoTemplate mongoTemplate) {
+        List<Document> manyList = new ArrayList<>();
+        for (DataMap dataMap : documentList) {
+            manyList.add(dataMap.formHash());
+        }
+        InsertManyOptions options = new InsertManyOptions();
+        options.ordered(false);
+        try {
+            mongoTemplate.getCollection("data").insertMany(manyList, options);
+        } catch (MongoBulkWriteException | DuplicateKeyException ignored) {
+        }
     }
 
-    return stringList;
-  }
+    public void saveLines(
+            Collection<DataMap> documentList, String sortField, MongoTemplate mongoTemplate) {
+        List<Document> manyList = new ArrayList<>();
+        for (DataMap dataMap : documentList) {
+            manyList.add(dataMap.formHash());
+        }
+        // 按照某个字段倒序排序
+        manyList.sort(
+                new Comparator<Document>() {
+                    @Override
+                    public int compare(Document o1, Document o2) {
+                        return (Double.valueOf(String.valueOf(o2.get(sortField)))
+                                .compareTo(Double.valueOf(String.valueOf(o1.get(sortField)))));
+                    }
+                });
+        InsertManyOptions options = new InsertManyOptions();
+        options.ordered(false);
+        try {
+            List<Document> batchInsertList = new ArrayList<>();
+            int k = 0;
+            for (Document document : manyList) {
+                batchInsertList.add(document);
+                k++;
+                if (k >= 1000) {
+                    mongoTemplate.getCollection("data").insertMany(batchInsertList, options);
+                    batchInsertList = new ArrayList<>();
+                    k = 0;
+                }
+            }
+            if (batchInsertList.size() > 0) {
+                mongoTemplate.getCollection("data").insertMany(batchInsertList, options);
+            }
+        } catch (Exception exception) {
+            exception.printStackTrace();
+        }
+    }
 
-  public static String getString(Document doc, Fields fields, String chineseHead) {
-    LinkedHashMap<String, String> inputReversed = fields.getFieldsReverse();
-    String alias = inputReversed.get(chineseHead);
-    return StringUtils.isNotBlank(alias) ? doc.get(alias, "") : "";
-  }
+    protected String createLogicAndDeleteOld(
+            String filename,
+            String modelId,
+            List<String> newHeaders,
+            MongoTemplate mongoTemplate,
+            FileService fileService) {
+        // 模糊匹配出所有的文件并删除
+        Query query =
+                new Query(
+                        Criteria.where("modelId")
+                                .is(modelId)
+                                .and("filename")
+                                .is(filename)
+                                .and("generated")
+                                .is(Boolean.TRUE));
+        List<LogicalFile> oldFiles = mongoTemplate.find(query, LogicalFile.class);
+        for (LogicalFile old : oldFiles) {
+            IdInput idInput = new IdInput();
+            idInput.setId(old.getId());
+            fileService.deleteById(idInput);
+        }
+        // 先删除原来的文件再创建
+        LogicalFile logicalFile = new LogicalFile();
+        logicalFile.setFilename(filename);
+        logicalFile.setStatus(FileStatus.STATUS_FINISHED);
+        String current = DataUtils.currentDate();
+        logicalFile.setUploadDate(current);
+        logicalFile.setModelId(modelId);
+        logicalFile.setGenerated(Boolean.TRUE);
+        // 插入文件获取文件id
+        logicalFile = mongoTemplate.insert(logicalFile);
+        Fields fields = new Fields();
+        fields.setFileId(logicalFile.getId());
+        // 插入表头
+        LinkedHashMap<String, String> header = new LinkedHashMap<>();
+        LinkedHashMap<String, String> headerReversed = new LinkedHashMap<>();
+        for (String field : newHeaders) {
+            header.put(field, field);
+            headerReversed.put(field, field);
+        }
+        fields.setFields(header);
+        fields.setFieldsReverse(headerReversed);
+        mongoTemplate.insert(fields);
+        return logicalFile.getId();
+    }
 
-  public static void setString(DataMap doc, String chineseHead, Fields fields, String value) {
-    LinkedHashMap<String, String> inputReversed = fields.getFieldsReverse();
-    String alias = inputReversed.get(chineseHead);
-    if (StringUtils.isNotBlank(alias)) {
-      doc.put(alias, value);
+    protected List<Fields> matchHeaders(List<Fields> needToMatch, List<String> headers) {
+        List<Fields> matched = new ArrayList<>();
+        for (Fields fields : needToMatch) {
+            List<String> oriHeaders = new ArrayList<>(fields.getFieldsReverse().keySet());
+            // 字段完全一样就说明是需要查找的表
+            if (headers.containsAll(oriHeaders) && oriHeaders.containsAll(headers)) {
+                matched.add(fields);
+            }
+        }
+        return matched;
     }
-  }
 
-  protected List<Fields> matchHeaders(List<Fields> needToMatch, List<String> headers) {
-    List<Fields> matched = new ArrayList<>();
-    for (Fields fields : needToMatch) {
-      List<String> oriHeaders = new ArrayList<>(fields.getFieldsReverse().keySet());
-      // 字段完全一样就说明是需要查找的表
-      if (headers.containsAll(oriHeaders) && oriHeaders.containsAll(headers)) {
-        matched.add(fields);
-      }
+    protected String getFileId(String filename, String modelId, MongoTemplate mongoTemplate) {
+        Query query =
+                new Query(
+                        Criteria.where("modelId")
+                                .is(modelId)
+                                .and("filename")
+                                .is(filename)
+                                .and("generated")
+                                .is(Boolean.TRUE));
+        LogicalFile oldFile = mongoTemplate.findOne(query, LogicalFile.class);
+        if (oldFile != null) {
+            return oldFile.getId();
+        } else {
+            return "";
+        }
     }
-    return matched;
-  }
 
-  protected String getFileId(String filename, String modelId, MongoTemplate mongoTemplate) {
-    Query query =
-        new Query(
-            Criteria.where("modelId")
-                .is(modelId)
-                .and("filename")
-                .is(filename)
-                .and("generated")
-                .is(Boolean.TRUE));
-    LogicalFile oldFile = mongoTemplate.findOne(query, LogicalFile.class);
-    if (oldFile != null) {
-      return oldFile.getId();
-    } else {
-      return "";
+
+    protected String getLogicalFileId(String filename, String modelId, MongoTemplate mongoTemplate) {
+        Query query =
+                new Query(
+                        Criteria.where("modelId")
+                                .is(modelId)
+                                .and("matchType")
+                                .is(filename));
+        LogicalFile oldFile = mongoTemplate.findOne(query, LogicalFile.class);
+        if (oldFile != null) {
+            return oldFile.getId();
+        } else {
+            return "";
+        }
     }
-  }
 
-  protected void insertFromTo(
-      String fromFileId, String toFileId, String modelId, MongoTemplate mongoTemplate) {
-    TFromToFileIds tFromToFileIds = new TFromToFileIds();
-    tFromToFileIds.setModelId(modelId);
-    tFromToFileIds.setFromFileId(fromFileId);
-    tFromToFileIds.setToFileId(toFileId);
-    mongoTemplate.insert(tFromToFileIds);
-  }
+    protected void insertFromTo(
+            String fromFileId, String toFileId, String modelId, MongoTemplate mongoTemplate) {
+        TFromToFileIds tFromToFileIds = new TFromToFileIds();
+        tFromToFileIds.setModelId(modelId);
+        tFromToFileIds.setFromFileId(fromFileId);
+        tFromToFileIds.setToFileId(toFileId);
+        mongoTemplate.insert(tFromToFileIds);
+    }
 }

src/main/java/ieven/server/webapp/service/alipay/AlipayService.java

@@ -32,206 +32,215 @@ import java.util.concurrent.Executors;
 @Slf4j
 @Service
 public class AlipayService {
-  @Autowired private MongoTemplate mongoTemplate;
-  @Autowired private HeaderProperties headerProperties;
-  @Autowired private FileService fileService;
+    @Autowired
+    FieldsService fieldsService;
+    @Autowired
+    private MongoTemplate mongoTemplate;
+    @Autowired
+    private HeaderProperties headerProperties;
+    @Autowired
+    private FileService fileService;
 
-  @Autowired FieldsService fieldsService;
-
-  public Mapped extractAll(String modelId) throws InterruptedException {
-    final List<Fields> originFields = queryAllFields(modelId);
-    // 先校验文件结构如果有不符合结构的文件 则暂停执行 给于提示
-    fieldsService.checkAllFields(originFields);
-    List<LogicalFile> logicalFiles = this.queryAllLogicalFiles2(modelId);
-    List<String> checkResultStrings = new ArrayList<>();
-    for (LogicalFile logicalFile : logicalFiles) {
-      if (StringUtils.isBlank(logicalFile.getMatchType())) {
-        checkResultStrings.add(logicalFile.getFilename() + "未匹配到文件校验规则,请检查!");
-      }
-    }
-    if (checkResultStrings.size() > 0) {
-      return Mapped.ERROR(String.join(" ", checkResultStrings));
-    }
-    // 先删除所有原来生成的表
-    clearAllGenerated(modelId);
-    // 查询出modelId的所有原始表头
-
-    if (CollectionUtils.isEmpty(originFields)) {
-      return Mapped.ERROR("文件为空");
-    }
+    public Mapped extractAll(String modelId) throws InterruptedException {
+        final List<Fields> originFields = queryAllFields(modelId);
+        // 先校验文件结构如果有不符合结构的文件 则暂停执行 给于提示
+        fieldsService.checkAllFields(originFields);
+        List<LogicalFile> logicalFiles = this.queryAllLogicalFiles2(modelId);
+        List<String> checkResultStrings = new ArrayList<>();
+        for (LogicalFile logicalFile : logicalFiles) {
+            if (StringUtils.isBlank(logicalFile.getMatchType())) {
+                checkResultStrings.add(logicalFile.getFilename() + "未匹配到文件校验规则,请检查!");
+            }
+        }
+        if (checkResultStrings.size() > 0) {
+            return Mapped.ERROR(String.join(" ", checkResultStrings));
+        }
+        // 先删除所有原来生成的表
+        clearAllGenerated(modelId);
+        // 查询出modelId的所有原始表头
+        if (CollectionUtils.isEmpty(originFields)) {
+            return Mapped.ERROR("文件为空");
+        }
 
-    // 无限制大小的threadpool
-    ExecutorService exec = Executors.newCachedThreadPool();
-    log.info("latch");
-    CountDownLatch latch = new CountDownLatch(7);
-    // 五联单-注册信息
-    exec.execute(new ALIPAY_ORIGIN_REGISTER(originFields, modelId, mongoTemplate, latch));
-    // 五联单-登录日志
-    exec.execute(new ALIPAY_GENERATED_LOGIN(originFields, modelId, mongoTemplate, latch));
-    // 五联单-交易记录
-    exec.execute(new ALIPAY_GENERATED_TRADE(originFields, modelId, mongoTemplate, latch));
-    // 五联单-账户明细
-    exec.execute(new ALIPAY_GENERATED_ACCOUNT(originFields, modelId, mongoTemplate, latch));
-    // 五联单-转账明细
-    exec.execute(new ALIPAY_GENERATED_TRANSFER(originFields, modelId, mongoTemplate, latch));
-    // 反诈平台订单数据
-    exec.execute(
-        new AntiCheat(headerProperties, originFields, modelId, mongoTemplate, fileService, latch));
-    // 治安平台订单数据
-    exec.execute(
-        new SecurityOrder(
-            headerProperties, originFields, modelId, mongoTemplate, fileService, latch));
-    latch.await();
-    log.info("latch");
-    log.info("latch2");
-    CountDownLatch latch2 = new CountDownLatch(7);
-    // 五联单-账户透视
-    exec.execute(
-        new ScaleAnalysis(
-            headerProperties, originFields, modelId, mongoTemplate, fileService, latch2));
-    // 转账分析
-    exec.execute(
-        new TransferAnalysis(
-            headerProperties, originFields, modelId, mongoTemplate, fileService, latch2));
-    // 对手分析
-    exec.execute(
-        new EndAnalysis(
-            headerProperties, originFields, modelId, mongoTemplate, fileService, latch2));
-    // 收货地址分析
-    exec.execute(
-        new DeliveryAnalysis(
-            headerProperties, originFields, modelId, mongoTemplate, fileService, latch2));
-    // IP分析
-    exec.execute(
-        new IpAnalysis(
-            headerProperties, originFields, modelId, mongoTemplate, fileService, latch2));
-    // 存储了账号-户名的的信息
-    Map<String, String> accountToName = new ConcurrentHashMap<>();
-    // 反诈治安交易数据明细-从反诈读取户名
-    exec.execute(new AntiSecurityTrade1(modelId, mongoTemplate, accountToName, latch2));
-    // 反诈治安交易数据明细-从治安读取户名
-    exec.execute(new AntiSecurityTrade2(modelId, mongoTemplate, accountToName, latch2));
-    latch2.await();
-    log.info("latch2");
-    log.info("latch3");
-    CountDownLatch latch3 = new CountDownLatch(1);
-    // 反诈治安交易数据明细-从反诈读取   反诈治安交易数据明细合并到这里
-    exec.execute(
-        new AntiSecurityTrade3(
-            headerProperties, modelId, mongoTemplate, accountToName, fileService, latch3));
-    latch3.await();
+        // 无限制大小的threadpool
+        ExecutorService exec = Executors.newCachedThreadPool();
+        log.info("latch");
+        CountDownLatch latch = new CountDownLatch(7);
+        // 五联单-注册信息
+        exec.execute(new ALIPAY_ORIGIN_REGISTER(originFields, modelId, mongoTemplate, latch));
+        // 五联单-登录日志
+        exec.execute(new ALIPAY_GENERATED_LOGIN(originFields, modelId, mongoTemplate, latch));
+        // 五联单-交易记录
+        exec.execute(new ALIPAY_GENERATED_TRADE(originFields, modelId, mongoTemplate, latch));
+        // 五联单-账户明细
+        exec.execute(new ALIPAY_GENERATED_ACCOUNT(originFields, modelId, mongoTemplate, latch));
+        // 五联单-转账明细
+        exec.execute(new ALIPAY_GENERATED_TRANSFER(originFields, modelId, mongoTemplate, latch));
+        // 反诈平台订单数据
+        exec.execute(
+                new AntiCheat(headerProperties, originFields, modelId, mongoTemplate, fileService, latch));
+        // 治安平台订单数据
+        exec.execute(
+                new SecurityOrder(
+                        headerProperties, originFields, modelId, mongoTemplate, fileService, latch));
+        latch.await();
+        log.info("latch");
+        log.info("latch2");
+        CountDownLatch latch2 = new CountDownLatch(7);
+        // 五联单-账户透视
+        exec.execute(
+                new ScaleAnalysis(
+                        headerProperties, originFields, modelId, mongoTemplate, fileService, latch2));
+        // 转账分析
+        exec.execute(
+                new TransferAnalysis(
+                        headerProperties, originFields, modelId, mongoTemplate, fileService, latch2));
+        // 对手分析
+        exec.execute(
+                new EndAnalysis(
+                        headerProperties, originFields, modelId, mongoTemplate, fileService, latch2));
+        // 收货地址分析
+        exec.execute(
+                new DeliveryAnalysis(
+                        headerProperties, originFields, modelId, mongoTemplate, fileService, latch2));
+        // IP分析
+        exec.execute(
+                new IpAnalysis(
+                        headerProperties, originFields, modelId, mongoTemplate, fileService, latch2));
+        // 存储了账号-户名的的信息
+        Map<String, String> accountToName = new ConcurrentHashMap<>();
+        // 反诈治安交易数据明细-从反诈读取户名
+        exec.execute(new AntiSecurityTrade1(modelId, mongoTemplate, accountToName, latch2));
+        // 反诈治安交易数据明细-从治安读取户名
+        exec.execute(new AntiSecurityTrade2(modelId, mongoTemplate, accountToName, latch2));
+        latch2.await();
+        log.info("latch2");
+        log.info("latch3");
+        CountDownLatch latch3 = new CountDownLatch(1);
+        // 反诈治安交易数据明细-从反诈读取   反诈治安交易数据明细合并到这里
+        exec.execute(
+                new AntiSecurityTrade3(
+                        headerProperties, modelId, mongoTemplate, accountToName, fileService, latch3));
+        latch3.await();
 
-    // 反诈治安交易数据明细-从治安读取
+        // 反诈治安交易数据明细-从治安读取
     /*CountDownLatch latch8 = new CountDownLatch(1);
     exec.execute(
         new AntiSecurityTrade4(
             headerProperties, modelId, mongoTemplate, accountToName, fileService, latch8));
     latch8.await();*/
-    log.info("latch3");
-    log.info("latch4");
-    CountDownLatch latch4 = new CountDownLatch(2);
-    // 反诈治安-账户透视
-    exec.execute(
-        new AntiSecurityTradeScaleAnalysis(
-            headerProperties, modelId, mongoTemplate, fileService, latch4));
-    // 反诈治安-对手分析
-    exec.execute(
-        new AntiSecurityTradeEndAnalysis(
-            headerProperties, modelId, mongoTemplate, fileService, latch4));
-    latch4.await();
-    log.info("latch4");
-    log.info("latch5");
-    CountDownLatch latch5 = new CountDownLatch(1);
-    // 财付通 开户信息
-    exec.execute(
-        new TenpayAccount(
-            originFields, modelId, mongoTemplate, fileService, accountToName, latch5));
-    latch5.await();
-    log.info("latch5");
-    log.info("latch6");
-    CountDownLatch latch6 = new CountDownLatch(1);
-    // 财付通-订单明细
-    exec.execute(new TenpayOrders(originFields, modelId, mongoTemplate, fileService, latch6));
-    latch6.await();
-    log.info("latch6");
-    log.info("latch7");
-    CountDownLatch latch9 = new CountDownLatch(1);
-    // 财付通-交易明细
-    exec.execute(new TenpayTrades(modelId, mongoTemplate, accountToName, fileService, latch9));
-    latch9.await();
-    CountDownLatch latch7 = new CountDownLatch(3);
-    // 财付通-手机明细
-    exec.execute(new TenpayMobile(originFields, modelId, mongoTemplate, fileService, latch7));
-    // 财付通-对手透视
-    exec.execute(
-        new TenpayTradesEndAnalysis(headerProperties, modelId, mongoTemplate, fileService, latch7));
-    // 财付通-账户透视
-    exec.execute(
-        new TenpayTradesScaleAnalysis(
-            headerProperties, modelId, mongoTemplate, fileService, latch7));
-    latch7.await();
-    reorder(modelId);
-    log.info("执行完毕");
-    return Mapped.OK();
-  }
+        log.info("latch3");
+        log.info("latch4");
+        CountDownLatch latch4 = new CountDownLatch(2);
+        // 反诈治安-账户透视
+        exec.execute(
+                new AntiSecurityTradeScaleAnalysis(
+                        headerProperties, modelId, mongoTemplate, fileService, latch4));
+        // 反诈治安-对手分析
+        exec.execute(
+                new AntiSecurityTradeEndAnalysis(
+                        headerProperties, modelId, mongoTemplate, fileService, latch4));
+        latch4.await();
+        log.info("latch4");
+        log.info("latch5");
+        CountDownLatch latch5 = new CountDownLatch(1);
+        // 财付通 开户信息
+        exec.execute(
+                new TenpayAccount(
+                        originFields, modelId, mongoTemplate, fileService, accountToName, latch5));
+        latch5.await();
+        log.info("latch5");
+        log.info("latch6");
+        CountDownLatch latch6 = new CountDownLatch(1);
+        // 财付通-订单明细
+        exec.execute(new TenpayOrders(originFields, modelId, mongoTemplate, fileService, latch6));
+        latch6.await();
+        log.info("latch6");
+        log.info("latch7");
+        CountDownLatch latch9 = new CountDownLatch(1);
+        // 财付通-交易明细
+        exec.execute(new TenpayTrades(modelId, mongoTemplate, accountToName, fileService, latch9));
+        latch9.await();
 
-  // 对文件重排序
-  private void reorder(String modelId) {
-    List<String> orderedNames = headerProperties.getOrderedNames();
-    Query query =
-        new Query(Criteria.where("modelId").is(modelId).and("generated").is(Boolean.TRUE));
-    List<LogicalFile> generated = mongoTemplate.find(query, LogicalFile.class);
-    // current
-    Date current = new Date();
-    long currentMills = current.getTime();
-    for (int i = 0; i < orderedNames.size(); i++) {
-      for (LogicalFile logicalFile : generated) {
-        String filename = logicalFile.getFilename();
-        if (orderedNames.get(i).equals(filename)) {
-          // 减1秒,造成顺序递减
-          current.setTime(currentMills - (long) i * 1000);
-          Update update = new Update().set("uploadDate", DataUtils.getDate(current));
-          String id = logicalFile.getId();
-          mongoTemplate.updateFirst(
-              new Query(Criteria.where("id").is(id)), update, LogicalFile.class);
-          break;
+        log.info("latch8");
+        CountDownLatch latch8 = new CountDownLatch(1);
+        // 财付通-订单明细
+        exec.execute(new TenpayTrades0422(originFields,modelId, mongoTemplate, accountToName, fileService, latch8));
+        latch8.await();
+        log.info("latch8");
+
+        CountDownLatch latch7 = new CountDownLatch(3);
+        // 财付通-手机明细
+        exec.execute(new TenpayMobile(originFields, modelId, mongoTemplate, fileService, latch7));
+        // 财付通-对手透视
+        exec.execute(
+                new TenpayTradesEndAnalysis(headerProperties, modelId, mongoTemplate, fileService, latch7));
+        // 财付通-账户透视
+        exec.execute(
+                new TenpayTradesScaleAnalysis(
+                        headerProperties, modelId, mongoTemplate, fileService, latch7));
+        latch7.await();
+        reorder(modelId);
+        log.info("执行完毕");
+        return Mapped.OK();
+    }
+
+    // 对文件重排序
+    private void reorder(String modelId) {
+        List<String> orderedNames = headerProperties.getOrderedNames();
+        Query query =
+                new Query(Criteria.where("modelId").is(modelId).and("generated").is(Boolean.TRUE));
+        List<LogicalFile> generated = mongoTemplate.find(query, LogicalFile.class);
+        // current
+        Date current = new Date();
+        long currentMills = current.getTime();
+        for (int i = 0; i < orderedNames.size(); i++) {
+            for (LogicalFile logicalFile : generated) {
+                String filename = logicalFile.getFilename();
+                if (orderedNames.get(i).equals(filename)) {
+                    // 减1秒,造成顺序递减
+                    current.setTime(currentMills - (long) i * 1000);
+                    Update update = new Update().set("uploadDate", DataUtils.getDate(current));
+                    String id = logicalFile.getId();
+                    mongoTemplate.updateFirst(
+                            new Query(Criteria.where("id").is(id)), update, LogicalFile.class);
+                    break;
+                }
+            }
         }
-      }
     }
-  }
 
-  public List<LogicalFile> queryAllLogicalFiles(String modelId) {
-    Query query =
-        new Query(Criteria.where("modelId").is(modelId).and("generated").is(Boolean.TRUE));
-    return mongoTemplate.find(query, LogicalFile.class);
-  }
+    public List<LogicalFile> queryAllLogicalFiles(String modelId) {
+        Query query =
+                new Query(Criteria.where("modelId").is(modelId).and("generated").is(Boolean.TRUE));
+        return mongoTemplate.find(query, LogicalFile.class);
+    }
 
-  public List<LogicalFile> queryAllLogicalFiles2(String modelId) {
-    Query query =
-        new Query(Criteria.where("modelId").is(modelId).and("generated").is(Boolean.FALSE));
-    return mongoTemplate.find(query, LogicalFile.class);
-  }
+    public List<LogicalFile> queryAllLogicalFiles2(String modelId) {
+        Query query =
+                new Query(Criteria.where("modelId").is(modelId).and("generated").is(Boolean.FALSE));
+        return mongoTemplate.find(query, LogicalFile.class);
+    }
 
-  private void clearAllGenerated(String modelId) {
-    List<LogicalFile> logicalFiles = queryAllLogicalFiles(modelId);
-    for (LogicalFile old : logicalFiles) {
-      IdInput idInput = new IdInput();
-      idInput.setId(old.getId());
-      fileService.deleteById(idInput);
+    private void clearAllGenerated(String modelId) {
+        List<LogicalFile> logicalFiles = queryAllLogicalFiles(modelId);
+        for (LogicalFile old : logicalFiles) {
+            IdInput idInput = new IdInput();
+            idInput.setId(old.getId());
+            fileService.deleteById(idInput);
+        }
+        mongoTemplate.remove(new Query(Criteria.where("modelId").is(modelId)), TFromToFileIds.class);
     }
-    mongoTemplate.remove(new Query(Criteria.where("modelId").is(modelId)), TFromToFileIds.class);
-  }
 
-  private List<Fields> queryAllFields(String modelId) {
-    Query query =
-        new Query(Criteria.where("modelId").is(modelId).and("generated").is(Boolean.FALSE));
-    List<LogicalFile> fileList = mongoTemplate.find(query, LogicalFile.class);
-    List<String> fileIdList = new ArrayList<>();
-    for (LogicalFile file : fileList) {
-      String fileId = file.getId();
-      fileIdList.add(fileId);
+    private List<Fields> queryAllFields(String modelId) {
+        Query query = new Query(Criteria.where("modelId").is(modelId).and("generated").is(Boolean.FALSE));
+        List<LogicalFile> fileList = mongoTemplate.find(query, LogicalFile.class);
+        List<String> fileIdList = new ArrayList<>();
+        for (LogicalFile file : fileList) {
+            String fileId = file.getId();
+            fileIdList.add(fileId);
+        }
+        Query query1 = new Query(Criteria.where("fileId").in(fileIdList));
+        return mongoTemplate.find(query1, Fields.class);
     }
-    Query query1 = new Query(Criteria.where("fileId").in(fileIdList));
-    return mongoTemplate.find(query1, Fields.class);
-  }
 }

src/main/java/ieven/server/webapp/service/tenpay/TenpayMobile.java

@@ -18,55 +18,55 @@ import java.util.concurrent.CountDownLatch;
 @Slf4j
 public class TenpayMobile extends Ops implements Runnable {
 
-  List<Fields> originFields;
-  String modelId;
-  MongoTemplate mongoTemplate;
-  FileService fileService;
-  CountDownLatch latch;
+    List<Fields> originFields;
+    String modelId;
+    MongoTemplate mongoTemplate;
+    FileService fileService;
+    CountDownLatch latch;
 
-  public TenpayMobile(
-      List<Fields> originFields,
-      String modelId,
-      MongoTemplate mongoTemplate,
-      FileService fileService,
-      CountDownLatch latch) {
-    this.originFields = originFields;
-    this.modelId = modelId;
-    this.mongoTemplate = mongoTemplate;
-    this.fileService = fileService;
-    this.latch = latch;
-  }
+    public TenpayMobile(
+            List<Fields> originFields,
+            String modelId,
+            MongoTemplate mongoTemplate,
+            FileService fileService,
+            CountDownLatch latch) {
+        this.originFields = originFields;
+        this.modelId = modelId;
+        this.mongoTemplate = mongoTemplate;
+        this.fileService = fileService;
+        this.latch = latch;
+    }
 
-  @Override
-  public void run() {
-    log.info("财付通-手机明细开始");
-    List<String> headers = TenpayHeaders.getHeaders(4);
-    // 直接删除旧表创建新表
-    List<Fields> matched = matchHeaders(originFields, headers);
-    if (CollectionUtils.isNotEmpty(matched)) {
-      String newFileId =
-          createLogicAndDeleteOld("财付通-手机明细", modelId, headers, mongoTemplate, fileService);
-      for (Fields fields : matched) {
-        String fileId = fields.getFileId();
-        MongoCursor<Document> cursor = getCursor(fileId, mongoTemplate);
-        Document originDoc;
-        List<DataMap> needToSave = new ArrayList<>(1000);
-        while (cursor.hasNext()) {
-          originDoc = cursor.next();
-          // 拷贝到新的
-          DataMap newDoc = initWithOrigin(originDoc, newFileId);
-          needToSave.add(newDoc);
-          if (needToSave.size() >= 1000) {
-            saveLines(new ArrayList<>(needToSave), mongoTemplate);
-            needToSave.clear();
-          }
-        }
-        if (CollectionUtils.isNotEmpty(needToSave)) {
-          saveLines(needToSave, mongoTemplate);
+    @Override
+    public void run() {
+        log.info("财付通-手机明细开始");
+        List<String> headers = TenpayHeaders.getHeaders(4);
+        // 直接删除旧表创建新表
+        List<Fields> matched = matchHeaders(originFields, headers);
+        if (CollectionUtils.isNotEmpty(matched)) {
+            String newFileId =
+                    createLogicAndDeleteOld("财付通-手机明细", modelId, headers, mongoTemplate, fileService);
+            for (Fields fields : matched) {
+                String fileId = fields.getFileId();
+                MongoCursor<Document> cursor = getCursor(fileId, mongoTemplate);
+                Document originDoc;
+                List<DataMap> needToSave = new ArrayList<>(1000);
+                while (cursor.hasNext()) {
+                    originDoc = cursor.next();
+                    // 拷贝到新的
+                    DataMap newDoc = initWithOrigin(originDoc, newFileId);
+                    needToSave.add(newDoc);
+                    if (needToSave.size() >= 1000) {
+                        saveLines(new ArrayList<>(needToSave), mongoTemplate);
+                        needToSave.clear();
+                    }
+                }
+                if (CollectionUtils.isNotEmpty(needToSave)) {
+                    saveLines(needToSave, mongoTemplate);
+                }
+            }
         }
-      }
+        latch.countDown();
+        log.info("财付通-手机明细结束");
     }
-    latch.countDown();
-    log.info("财付通-手机明细结束");
-  }
 }

src/main/java/ieven/server/webapp/service/tenpay/TenpayOrders.java

@@ -19,127 +19,125 @@ import java.util.concurrent.CountDownLatch;
 @Slf4j
 public class TenpayOrders extends Ops implements Runnable {
 
-  List<Fields> originFields;
-  String modelId;
-  MongoTemplate mongoTemplate;
-  FileService fileService;
+    List<Fields> originFields;
+    String modelId;
+    MongoTemplate mongoTemplate;
+    FileService fileService;
 
-  CountDownLatch latch;
+    CountDownLatch latch;
 
-  public TenpayOrders(
-      List<Fields> originFields,
-      String modelId,
-      MongoTemplate mongoTemplate,
-      FileService fileService,
-      CountDownLatch latch) {
-    this.originFields = originFields;
-    this.modelId = modelId;
-    this.mongoTemplate = mongoTemplate;
-    this.fileService = fileService;
-    this.latch = latch;
-  }
+    public TenpayOrders(
+            List<Fields> originFields,
+            String modelId,
+            MongoTemplate mongoTemplate,
+            FileService fileService,
+            CountDownLatch latch) {
+        this.originFields = originFields;
+        this.modelId = modelId;
+        this.mongoTemplate = mongoTemplate;
+        this.fileService = fileService;
+        this.latch = latch;
+    }
 
-  @Override
-  public void run() {
-    log.info("财付通-订单明细开始");
-    List<String> origheaders = TenpayHeaders.getHeaders(6);
-    List<String> headers = TenpayHeaders.getHeaders(1);
-    List<Fields> matched = matchHeaders(originFields, headers);
-    String newFileId = "";
-    if (CollectionUtils.isNotEmpty(matched)) {
-      newFileId =
-          createLogicAndDeleteOld("财付通-订单明细", modelId, origheaders, mongoTemplate, fileService);
-      for (Fields fields : matched) {
-        String fileId = fields.getFileId();
-        MongoCursor<Document> cursor = getCursor(fileId, mongoTemplate);
-        Document originDoc;
-        List<DataMap> needToSave = new ArrayList<>(1000);
-        while (cursor.hasNext()) {
-          originDoc = cursor.next();
-          // 拷贝到新的
-          DataMap newDoc = initWithOrigin(originDoc, newFileId);
-          String s = (String) newDoc.get("用户ID");
-          if (StringUtils.isNotBlank(s) && s.contains("[")) {
-            s = s.replace("[", "").replace("]", "");
-            newDoc.put("用户ID", s);
-          }
-          String s2 = (String) newDoc.get("交易单号");
-          if (StringUtils.isNotBlank(s2) && s2.contains("[")) {
-            s2 = s2.replace("[", "").replace("]", "");
-            newDoc.put("交易单号", s2);
-          }
-          needToSave.add(newDoc);
-          if (needToSave.size() >= 1000) {
-            saveLines(new ArrayList<>(needToSave), mongoTemplate);
-            needToSave.clear();
-          }
-        }
-        if (CollectionUtils.isNotEmpty(needToSave)) {
-          saveLines(needToSave, mongoTemplate);
+    @Override
+    public void run() {
+        log.info("财付通-订单明细开始");
+        List<String> origheaders = TenpayHeaders.getHeaders(6);
+        List<String> headers = TenpayHeaders.getHeaders(1);
+        List<Fields> matched = matchHeaders(originFields, headers);
+        String newFileId = "";
+        if (CollectionUtils.isNotEmpty(matched)) {
+            newFileId = createLogicAndDeleteOld("财付通-订单明细", modelId, origheaders, mongoTemplate, fileService);
+            for (Fields fields : matched) {
+                String fileId = fields.getFileId();
+                MongoCursor<Document> cursor = getCursor(fileId, mongoTemplate);
+                Document originDoc;
+                List<DataMap> needToSave = new ArrayList<>(1000);
+                while (cursor.hasNext()) {
+                    originDoc = cursor.next();
+                    // 拷贝到新的
+                    DataMap newDoc = initWithOrigin(originDoc, newFileId);
+                    String s = (String) newDoc.get("用户ID");
+                    if (StringUtils.isNotBlank(s) && s.contains("[")) {
+                        s = s.replace("[", "").replace("]", "");
+                        newDoc.put("用户ID", s);
+                    }
+                    String s2 = (String) newDoc.get("交易单号");
+                    if (StringUtils.isNotBlank(s2) && s2.contains("[")) {
+                        s2 = s2.replace("[", "").replace("]", "");
+                        newDoc.put("交易单号", s2);
+                    }
+                    needToSave.add(newDoc);
+                    if (needToSave.size() >= 1000) {
+                        saveLines(new ArrayList<>(needToSave), mongoTemplate);
+                        needToSave.clear();
+                    }
+                }
+                if (CollectionUtils.isNotEmpty(needToSave)) {
+                    saveLines(needToSave, mongoTemplate);
+                }
+            }
         }
-      }
-    }
 
-    headers = TenpayHeaders.getHeaders(5);
-    matched = matchHeaders(originFields, headers);
+        headers = TenpayHeaders.getHeaders(5);
+        matched = matchHeaders(originFields, headers);
 
-    // 以下为处理另一种格式的财付通数据
-    if (CollectionUtils.isNotEmpty(matched)) {
-      if (StringUtils.isBlank(newFileId)) {
-        newFileId =
-            createLogicAndDeleteOld("财付通-订单明细", modelId, origheaders, mongoTemplate, fileService);
-      }
-      for (Fields fields : matched) {
-        String fileId = fields.getFileId();
-        MongoCursor<Document> cursor = getCursor(fileId, mongoTemplate);
-        Document originDoc;
-        List<DataMap> needToSave = new ArrayList<>(1000);
-        while (cursor.hasNext()) {
-          originDoc = cursor.next();
-          // 拷贝到新的
-          // 用户ID 交易单号 借贷类型 交易类型 交易金额（分） 账户余额（分） 交易时间 银行类型 交易说明 商户名称 发送方 发送金额（分） 接收方 接收时间 接收金额（分）
-          // 银行卡号 网银联单号1 网银联单号2
-          // DataMap newDoc = initWithOrigin(originDoc, newFileId);
-          DataMap newDoc = new DataMap();
-          newDoc.put("fileId", newFileId);
-          newDoc.put("用户ID", originDoc.getString("用户ID"));
-          newDoc.put("交易单号", originDoc.getString("交易单号"));
-          String f1 = originDoc.getString("借贷类型");
-          newDoc.put("借贷类型", originDoc.getString("借贷类型"));
-          newDoc.put("交易类型", originDoc.getString("交易用途类型"));
-          newDoc.put("交易金额(分)", originDoc.getString("交易金额(分)"));
-          newDoc.put("账户余额(分)", originDoc.getString("账户余额(分)"));
-          newDoc.put("交易时间", originDoc.getString("交易时间"));
-          newDoc.put("银行类型", originDoc.getString("交易用途类型"));
-          newDoc.put("交易说明", originDoc.getString("备注1") + originDoc.getString("备注2"));
-          if (f1.equals("出")) {
-            newDoc.put("发送金额(分)", originDoc.getString("交易金额(分)"));
-            newDoc.put("银行卡号", originDoc.getString("用户银行卡号"));
-            newDoc.put("发送方", originDoc.getString("用户ID"));
-            newDoc.put("接收方", originDoc.getString("对手方ID"));
-            newDoc.put("对手银行卡号", originDoc.getString("对手方银行卡号"));
-          }
-          if (f1.equals("入")) {
-            newDoc.put("接收金额(分)", originDoc.getString("交易金额(分)"));
-            newDoc.put("银行卡号", originDoc.getString("对手方银行卡号"));
-            newDoc.put("发送方", originDoc.getString("对手方ID"));
-            newDoc.put("接收方", originDoc.getString("用户ID"));
-            newDoc.put("对手银行卡号", originDoc.getString("用户银行卡号"));
-          }
-          newDoc.put("网银联单号1", originDoc.getString("用户侧网银联单号"));
-          newDoc.put("网银联单号2", originDoc.getString("对手侧网银联单号"));
-          needToSave.add(newDoc);
-          if (needToSave.size() >= 1000) {
-            saveLines(new ArrayList<>(needToSave), mongoTemplate);
-            needToSave.clear();
-          }
-        }
-        if (CollectionUtils.isNotEmpty(needToSave)) {
-          saveLines(needToSave, mongoTemplate);
+        // 以下为处理另一种格式的财付通数据
+        if (CollectionUtils.isNotEmpty(matched)) {
+            if (StringUtils.isBlank(newFileId)) {
+                newFileId = createLogicAndDeleteOld("财付通-订单明细", modelId, origheaders, mongoTemplate, fileService);
+            }
+            for (Fields fields : matched) {
+                String fileId = fields.getFileId();
+                MongoCursor<Document> cursor = getCursor(fileId, mongoTemplate);
+                Document originDoc;
+                List<DataMap> needToSave = new ArrayList<>(1000);
+                while (cursor.hasNext()) {
+                    originDoc = cursor.next();
+                    // 拷贝到新的
+                    // 用户ID 交易单号 借贷类型 交易类型 交易金额（分） 账户余额（分） 交易时间 银行类型 交易说明 商户名称 发送方 发送金额（分） 接收方 接收时间 接收金额（分）
+                    // 银行卡号 网银联单号1 网银联单号2
+                    // DataMap newDoc = initWithOrigin(originDoc, newFileId);
+                    DataMap newDoc = new DataMap();
+                    newDoc.put("fileId", newFileId);
+                    newDoc.put("用户ID", originDoc.getString("用户ID"));
+                    newDoc.put("交易单号", originDoc.getString("交易单号"));
+                    String f1 = originDoc.getString("借贷类型");
+                    newDoc.put("借贷类型", originDoc.getString("借贷类型"));
+                    newDoc.put("交易类型", originDoc.getString("交易用途类型"));
+                    newDoc.put("交易金额(分)", originDoc.getString("交易金额(分)"));
+                    newDoc.put("账户余额(分)", originDoc.getString("账户余额(分)"));
+                    newDoc.put("交易时间", originDoc.getString("交易时间"));
+                    newDoc.put("银行类型", originDoc.getString("交易用途类型"));
+                    newDoc.put("交易说明", originDoc.getString("备注1") + originDoc.getString("备注2"));
+                    if (f1.equals("出")) {
+                        newDoc.put("发送金额(分)", originDoc.getString("交易金额(分)"));
+                        newDoc.put("银行卡号", originDoc.getString("用户银行卡号"));
+                        newDoc.put("发送方", originDoc.getString("用户ID"));
+                        newDoc.put("接收方", originDoc.getString("对手方ID"));
+                        newDoc.put("对手银行卡号", originDoc.getString("对手方银行卡号"));
+                    }
+                    if (f1.equals("入")) {
+                        newDoc.put("接收金额(分)", originDoc.getString("交易金额(分)"));
+                        newDoc.put("银行卡号", originDoc.getString("对手方银行卡号"));
+                        newDoc.put("发送方", originDoc.getString("对手方ID"));
+                        newDoc.put("接收方", originDoc.getString("用户ID"));
+                        newDoc.put("对手银行卡号", originDoc.getString("用户银行卡号"));
+                    }
+                    newDoc.put("网银联单号1", originDoc.getString("用户侧网银联单号"));
+                    newDoc.put("网银联单号2", originDoc.getString("对手侧网银联单号"));
+                    needToSave.add(newDoc);
+                    if (needToSave.size() >= 1000) {
+                        saveLines(new ArrayList<>(needToSave), mongoTemplate);
+                        needToSave.clear();
+                    }
+                }
+                if (CollectionUtils.isNotEmpty(needToSave)) {
+                    saveLines(needToSave, mongoTemplate);
+                }
+            }
         }
-      }
+        latch.countDown();
+        log.info("财付通-订单明细结束");
     }
-    latch.countDown();
-    log.info("财付通-订单明细结束");
-  }
 }

src/main/java/ieven/server/webapp/service/tenpay/TenpayTrades.java

@@ -19,144 +19,135 @@ import java.util.concurrent.CountDownLatch;
 
 @Slf4j
 public class TenpayTrades extends Ops implements Runnable {
-  MongoTemplate mongoTemplate;
-  String modelId;
+    MongoTemplate mongoTemplate;
+    String modelId;
 
-  Map<String, String> accountToName;
-  FileService fileService;
+    Map<String, String> accountToName;
+    FileService fileService;
 
-  CountDownLatch latch;
+    CountDownLatch latch;
 
-  public TenpayTrades(
-      String modelId,
-      MongoTemplate mongoTemplate,
-      Map<String, String> accountToName,
-      FileService fileService,
-      CountDownLatch latch) {
-    this.accountToName = accountToName;
-    this.modelId = modelId;
-    this.mongoTemplate = mongoTemplate;
-    this.fileService = fileService;
-    this.latch = latch;
-  }
+    public TenpayTrades(
+            String modelId,
+            MongoTemplate mongoTemplate,
+            Map<String, String> accountToName,
+            FileService fileService,
+            CountDownLatch latch) {
+        this.accountToName = accountToName;
+        this.modelId = modelId;
+        this.mongoTemplate = mongoTemplate;
+        this.fileService = fileService;
+        this.latch = latch;
+    }
 
-  @Override
-  public void run() {
-    log.info("财付通-交易明细开始");
-    String antiSecurityFileId = "";
+    @Override
+    public void run() {
+        log.info("财付通-交易明细开始");
+        String antiSecurityFileId = "";
 
-    String dateFileId = getFileId("财付通-订单明细", modelId, mongoTemplate);
-    // 遍历登录日志全表
-    if (StringUtils.isNotBlank(dateFileId)) {
-      MongoCursor<Document> cursor = getCursor(dateFileId, mongoTemplate);
-      if (cursor.hasNext()) {
-        antiSecurityFileId =
-            createLogicAndDeleteOld(
-                "财付通-交易明细", modelId, TenpayHeaders.getHeaders(2), mongoTemplate, fileService);
-      }
-      Document origin;
-      List<DataMap> needToSave = new ArrayList<>(1000);
-      while (cursor.hasNext()) {
-        origin = cursor.next();
-        DataMap dataMap = new DataMap();
-        dataMap.put("fileId", antiSecurityFileId);
-        String userId = origin.getString("用户ID").replace("[", "").replace("]", "");
-        dataMap.put("用户ID", userId);
-        String lognumber = origin.getString("交易单号");
-        if (StringUtils.isBlank(lognumber)) {
-          continue;
-        }
-        dataMap.put("交易单号", lognumber.replace("[", "").replace("]", ""));
-        String flag = origin.getString("借贷类型");
-        if ("出".equals(flag)) {
-          dataMap.put("交易主体账号", origin.getString("发送方"));
-          if (StringUtils.isBlank(origin.getString("发送方"))) {
-            dataMap.put("交易主体账号", userId);
-          }
-          dataMap.put("交易主体银行卡号", origin.getString("银行卡号"));
-          dataMap.put("交易主体银行名称", "");
-          dataMap.put("交易主体户名", accountToName.getOrDefault(origin.getString("发送方"), ""));
-          dataMap.put("交易对手账号", origin.getString("接收方"));
-          dataMap.put("交易对手银行卡号", origin.getString("对手银行卡号"));
-          dataMap.put("交易对手银行名称", "");
-          dataMap.put(
-              "交易对手户名",
-              accountToName.getOrDefault(StringUtils.trimToEmpty(origin.getString("接收方")), ""));
-        }
-        if ("入".equals(flag)) {
-          dataMap.put("交易主体账号", StringUtils.trimToEmpty(origin.getString("接收方")));
-          if (StringUtils.isBlank(origin.getString("接收方"))) {
-            dataMap.put("交易主体账号", userId);
-          }
-          dataMap.put("交易主体银行卡号", origin.getString("对手银行卡号"));
-          dataMap.put("交易主体银行名称", "");
-          dataMap.put(
-              "交易主体户名",
-              accountToName.getOrDefault(StringUtils.trimToEmpty(origin.getString("接收方")), ""));
-
-          dataMap.put("交易对手账号", StringUtils.trimToEmpty(origin.getString("发送方")));
-          dataMap.put("交易对手银行卡号", origin.getString("银行卡号"));
-          dataMap.put("交易对手银行名称", "");
-          dataMap.put(
-              "交易对手户名",
-              accountToName.getOrDefault(StringUtils.trimToEmpty(origin.getString("发送方")), ""));
-        }
+        String dateFileId = getFileId("财付通-订单明细", modelId, mongoTemplate);
+        // 遍历登录日志全表
+        if (StringUtils.isNotBlank(dateFileId)) {
+            MongoCursor<Document> cursor = getCursor(dateFileId, mongoTemplate);
+            if (cursor.hasNext()) {
+                antiSecurityFileId = createLogicAndDeleteOld("财付通-交易明细", modelId, TenpayHeaders.getHeaders(2), mongoTemplate, fileService);
+            }
+            Document origin;
+            List<DataMap> needToSave = new ArrayList<>(1000);
+            while (cursor.hasNext()) {
+                origin = cursor.next();
+                DataMap dataMap = new DataMap();
+                dataMap.put("fileId", antiSecurityFileId);
+                String userId = origin.getString("用户ID").replace("[", "").replace("]", "");
+                dataMap.put("用户ID", userId);
+                String lognumber = origin.getString("交易单号");
+                if (StringUtils.isBlank(lognumber)) {
+                    continue;
+                }
+                dataMap.put("交易单号", lognumber.replace("[", "").replace("]", ""));
+                String flag = origin.getString("借贷类型");
+                if ("出".equals(flag)) {
+                    dataMap.put("交易主体账号", origin.getString("发送方"));
+                    if (StringUtils.isBlank(origin.getString("发送方"))) {
+                        dataMap.put("交易主体账号", userId);
+                    }
+                    dataMap.put("交易主体银行卡号", origin.getString("银行卡号"));
+                    dataMap.put("交易主体银行名称", "");
+                    dataMap.put("交易主体户名", accountToName.getOrDefault(origin.getString("发送方"), ""));
+                    dataMap.put("交易对手账号", origin.getString("接收方"));
+                    dataMap.put("交易对手银行卡号", origin.getString("对手银行卡号"));
+                    dataMap.put("交易对手银行名称", "");
+                    dataMap.put("交易对手户名", accountToName.getOrDefault(StringUtils.trimToEmpty(origin.getString("接收方")), ""));
+                }
+                if ("入".equals(flag)) {
+                    dataMap.put("交易主体账号", StringUtils.trimToEmpty(origin.getString("接收方")));
+                    if (StringUtils.isBlank(origin.getString("接收方"))) {
+                        dataMap.put("交易主体账号", userId);
+                    }
+                    dataMap.put("交易主体银行卡号", origin.getString("对手银行卡号"));
+                    dataMap.put("交易主体银行名称", "");
+                    dataMap.put(
+                            "交易主体户名",
+                            accountToName.getOrDefault(StringUtils.trimToEmpty(origin.getString("接收方")), ""));
 
-        dataMap.put("借贷类型", flag);
-        dataMap.put("交易类型", origin.getString("交易类型"));
-        dataMap.put("交易时间", origin.getString("交易时间"));
-        String string5 = origin.getString("交易金额(分)");
-        if (!NumberUtil.isNumber(string5)) {
-          continue;
-        }
-        dataMap.put(
-            "交易金额(元)",
-            NumberUtil.roundStr(
-                NumberUtil.div(string5 == null ? "0" : string5, "100").doubleValue(), 2));
-        String string6 = origin.getString("账户余额(分)");
-        if (!NumberUtil.isNumber(string6)) {
-          continue;
-        }
-        dataMap.put(
-            "账户余额(元)",
-            NumberUtil.roundStr(
-                NumberUtil.div(string6 == null ? "0" : string6, "100").doubleValue(), 2));
-        dataMap.put("银行类型", origin.getString("银行类型"));
-        dataMap.put("交易说明", origin.getString("交易说明"));
-        dataMap.put("网银联单号1", origin.getString("网银联单号1"));
-        dataMap.put("网银联单号2", origin.getString("网银联单号2"));
+                    dataMap.put("交易对手账号", StringUtils.trimToEmpty(origin.getString("发送方")));
+                    dataMap.put("交易对手银行卡号", origin.getString("银行卡号"));
+                    dataMap.put("交易对手银行名称", "");
+                    dataMap.put(
+                            "交易对手户名",
+                            accountToName.getOrDefault(StringUtils.trimToEmpty(origin.getString("发送方")), ""));
+                }
 
-        if ("".equals(dataMap.get("交易主体户名"))) {
-          String account = (String) dataMap.getOrDefault("交易主体账号", "");
-          if (StringUtils.isBlank(account) || "-".equals(account)) {
-          } else {
-            String newName = accountToName.getOrDefault(account, "");
-            if (StringUtils.isNotBlank(newName)) {
-              dataMap.put("交易主体户名", newName);
+                dataMap.put("借贷类型", flag);
+                dataMap.put("交易类型", origin.getString("交易类型"));
+                dataMap.put("交易时间", origin.getString("交易时间"));
+                String string5 = origin.getString("交易金额(分)");
+                if (!NumberUtil.isNumber(string5)) {
+                    continue;
+                }
+                dataMap.put("交易金额(元)", NumberUtil.roundStr(NumberUtil.div(string5 == null ? "0" : string5, "100").doubleValue(), 2));
+                String string6 = origin.getString("账户余额(分)");
+                if (!NumberUtil.isNumber(string6)) {
+                    continue;
+                }
+                dataMap.put("账户余额(元)", NumberUtil.roundStr(NumberUtil.div(string6 == null ? "0" : string6, "100").doubleValue(), 2));
+                dataMap.put("银行类型", origin.getString("银行类型"));
+                dataMap.put("交易说明", origin.getString("交易说明"));
+                dataMap.put("网银联单号1", origin.getString("网银联单号1"));
+                dataMap.put("网银联单号2", origin.getString("网银联单号2"));
+                dataMap.put("交易主体第三方账号名", origin.getString("第三方账户名称"));
+                dataMap.put("对手第三方账号名", origin.getString("第三方账户名称"));
+                if ("".equals(dataMap.get("交易主体户名"))) {
+                    String account = (String) dataMap.getOrDefault("交易主体账号", "");
+                    if (StringUtils.isBlank(account) || "-".equals(account)) {
+                    } else {
+                        String newName = accountToName.getOrDefault(account, "");
+                        if (StringUtils.isNotBlank(newName)) {
+                            dataMap.put("交易主体户名", newName);
+                        }
+                    }
+                }
+                if ("".equals(dataMap.get("交易对手户名"))) {
+                    String account = (String) dataMap.getOrDefault("交易对手账号", "");
+                    if (StringUtils.isBlank(account) || "-".equals(account)) {
+                    } else {
+                        String newName = accountToName.getOrDefault(account, "");
+                        if (StringUtils.isNotBlank(newName)) {
+                            dataMap.put("交易对手户名", newName);
+                        }
+                    }
+                }
+                needToSave.add(dataMap);
+                if (needToSave.size() >= 1000) {
+                    saveLines(new ArrayList<>(needToSave), mongoTemplate);
+                    needToSave.clear();
+                }
             }
-          }
-        }
-        if ("".equals(dataMap.get("交易对手户名"))) {
-          String account = (String) dataMap.getOrDefault("交易对手账号", "");
-          if (StringUtils.isBlank(account) || "-".equals(account)) {
-          } else {
-            String newName = accountToName.getOrDefault(account, "");
-            if (StringUtils.isNotBlank(newName)) {
-              dataMap.put("交易对手户名", newName);
+            if (CollectionUtils.isNotEmpty(needToSave)) {
+                saveLines(needToSave, mongoTemplate);
             }
-          }
-        }
-        needToSave.add(dataMap);
-        if (needToSave.size() >= 1000) {
-          saveLines(new ArrayList<>(needToSave), mongoTemplate);
-          needToSave.clear();
         }
-      }
-      if (CollectionUtils.isNotEmpty(needToSave)) {
-        saveLines(needToSave, mongoTemplate);
-      }
+        this.latch.countDown();
+        log.info("财付通-交易明细结束");
     }
-    this.latch.countDown();
-    log.info("财付通-交易明细结束");
-  }
 }

src/main/java/ieven/server/webapp/service/tenpay/TenpayTrades0422.java

@@ -0,0 +1,183 @@
+package ieven.server.webapp.service.tenpay;
+
+import cn.hutool.core.util.NumberUtil;
+import com.mongodb.client.MongoCursor;
+import ieven.server.webapp.domain.data.DataMap;
+import ieven.server.webapp.domain.data.Fields;
+import ieven.server.webapp.domain.file.FileService;
+import ieven.server.webapp.domain.tenpay.TenpayHeaders;
+import ieven.server.webapp.service.Ops;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.bson.Document;
+import org.springframework.data.mongodb.core.MongoTemplate;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+
+@Slf4j
+public class TenpayTrades0422 extends Ops implements Runnable {
+    MongoTemplate mongoTemplate;
+    String modelId;
+
+    Map<String, String> accountToName;
+    FileService fileService;
+
+    CountDownLatch latch;
+    List<Fields> originFields;
+
+
+    public TenpayTrades0422(
+            List<Fields> originFields,
+            String modelId,
+            MongoTemplate mongoTemplate,
+            Map<String, String> accountToName,
+            FileService fileService,
+            CountDownLatch latch) {
+        this.accountToName = accountToName;
+        this.modelId = modelId;
+        this.mongoTemplate = mongoTemplate;
+        this.fileService = fileService;
+        this.latch = latch;
+        this.originFields = originFields;
+    }
+
+    @Override
+    public void run() {
+        log.info("财付通-交易明细开始");
+        String antiSecurityFileId = "";
+
+        List<String> headers = TenpayHeaders.getHeaders(7);
+        List<Fields> matched = matchHeaders(originFields, headers);
+        String newFileId = "";
+        if (CollectionUtils.isNotEmpty(matched)) {
+            newFileId = createLogicAndDeleteOld("财付通-订单明细", modelId, headers, mongoTemplate, fileService);
+            for (Fields fields : matched) {
+                String fileId = fields.getFileId();
+                MongoCursor<Document> cursor = getCursor(fileId, mongoTemplate);
+                Document originDoc;
+                List<DataMap> needToSave = new ArrayList<>(1000);
+                while (cursor.hasNext()) {
+                    originDoc = cursor.next();
+                    // 拷贝到新的
+                    DataMap newDoc = initWithOrigin(originDoc, newFileId);
+                    String s = (String) newDoc.get("用户ID");
+                    if (StringUtils.isNotBlank(s) && s.contains("[")) {
+                        s = s.replace("[", "").replace("]", "");
+                        newDoc.put("用户ID", s);
+                    }
+                    String s2 = (String) newDoc.get("交易单号");
+                    if (StringUtils.isNotBlank(s2) && s2.contains("[")) {
+                        s2 = s2.replace("[", "").replace("]", "");
+                        newDoc.put("交易单号", s2);
+                    }
+                    needToSave.add(newDoc);
+                    if (needToSave.size() >= 1000) {
+                        saveLines(new ArrayList<>(needToSave), mongoTemplate);
+                        needToSave.clear();
+                    }
+                }
+                if (CollectionUtils.isNotEmpty(needToSave)) {
+                    saveLines(needToSave, mongoTemplate);
+                }
+            }
+        }
+        String dateFileId = getLogicalFileId("财付通-订单明细", modelId, mongoTemplate);
+        // 遍历登录日志全表
+        if (StringUtils.isNotBlank(dateFileId)) {
+            MongoCursor<Document> cursor = getCursor(dateFileId, mongoTemplate);
+            if (cursor.hasNext()) {
+                antiSecurityFileId = createLogicAndDeleteOld("财付通-交易明细", modelId, TenpayHeaders.getHeaders(7), mongoTemplate, fileService);
+            }
+            Document origin;
+            List<DataMap> needToSave = new ArrayList<>(1000);
+            while (cursor.hasNext()) {
+                origin = cursor.next();
+                DataMap dataMap = new DataMap();
+                dataMap.put("fileId", antiSecurityFileId);
+                String userId = origin.getString("用户ID").replace("[", "").replace("]", "");
+                dataMap.put("用户ID", userId);
+                String lognumber = origin.getString("交易单号");
+                if (StringUtils.isBlank(lognumber)) {
+                    continue;
+                }
+                dataMap.put("交易单号", lognumber.replace("[", "").replace("]", ""));
+                String flag = origin.getString("借贷类型");
+                if ("出".equals(flag)) {
+                    dataMap.put("交易主体账号", origin.getString("用户ID"));
+                    dataMap.put("交易主体银行卡号", origin.getString("用户银行卡号"));
+                    dataMap.put("交易主体银行名称", origin.getString("用户侧银行名称"));
+                    //dataMap.put("交易主体户名", accountToName.getOrDefault(origin.getString("用户银行卡号"), ""));
+                    dataMap.put("交易主体户名", origin.getString("用户侧账号名称"));
+                    dataMap.put("交易对手账号", origin.getString("对手方ID"));
+                    dataMap.put("交易对手银行卡号", origin.getString("对手方银行卡号"));
+                    dataMap.put("交易对手银行名称", origin.getString("对手侧银行名称"));
+                    //dataMap.put("交易对手户名", accountToName.getOrDefault(StringUtils.trimToEmpty(origin.getString("对手侧账户名称")), ""));
+                    dataMap.put("交易对手户名", origin.getString("对手侧账户名称"));
+                }
+                if ("入".equals(flag)) {
+                    dataMap.put("交易主体账号", StringUtils.trimToEmpty(origin.getString("对手方ID")));
+                    dataMap.put("交易主体银行卡号", origin.getString("对手方银行卡号"));
+                    dataMap.put("交易主体银行名称", origin.getString("对手侧银行名称"));
+                    dataMap.put("交易主体户名", origin.getString("对手侧账户名称"));
+                    dataMap.put("交易对手账号", StringUtils.trimToEmpty(origin.getString("用户ID")));
+                    dataMap.put("交易对手银行卡号", origin.getString("用户银行卡号"));
+                    dataMap.put("交易对手银行名称", origin.getString("用户侧银行名称"));
+                    dataMap.put("交易对手户名",  origin.getString("用户侧账号名称"));
+                }
+                dataMap.put("借贷类型", flag);
+                dataMap.put("交易类型", origin.getString("交易用途类型"));
+                dataMap.put("交易时间", origin.getString("交易时间"));
+                String string5 = origin.getString("交易金额(分)");
+                if (!NumberUtil.isNumber(string5)) {
+                    continue;
+                }
+                dataMap.put("交易金额(元)", NumberUtil.roundStr(NumberUtil.div(string5 == null ? "0" : string5, "100").doubleValue(), 2));
+                String string6 = origin.getString("账户余额(分)");
+                if (!NumberUtil.isNumber(string6)) {
+                    continue;
+                }
+                dataMap.put("账户余额(元)", NumberUtil.roundStr(NumberUtil.div(string6 == null ? "0" : string6, "100").doubleValue(), 2));
+                dataMap.put("银行类型", origin.getString("交易业务类型"));
+                dataMap.put("交易说明", origin.getString("备注1"));
+                dataMap.put("网银联单号1", origin.getString("用户侧网银联单号"));
+                dataMap.put("网银联单号2", origin.getString("对手侧网银联单号"));
+                dataMap.put("交易主体第三方账号名", origin.getString("第三方账户名称"));
+                dataMap.put("对手第三方账号名", origin.getString("第三方账户名称"));
+                if ("".equals(dataMap.get("交易主体户名"))) {
+                    String account = (String) dataMap.getOrDefault("交易主体账号", "");
+                    if (StringUtils.isBlank(account) || "-".equals(account)) {
+                    } else {
+                        String newName = accountToName.getOrDefault(account, "");
+                        if (StringUtils.isNotBlank(newName)) {
+                            dataMap.put("交易主体户名", newName);
+                        }
+                    }
+                }
+                if ("".equals(dataMap.get("交易对手户名"))) {
+                    String account = (String) dataMap.getOrDefault("交易对手账号", "");
+                    if (StringUtils.isBlank(account) || "-".equals(account)) {
+                    } else {
+                        String newName = accountToName.getOrDefault(account, "");
+                        if (StringUtils.isNotBlank(newName)) {
+                            dataMap.put("交易对手户名", newName);
+                        }
+                    }
+                }
+                needToSave.add(dataMap);
+                if (needToSave.size() >= 1000) {
+                    saveLines(new ArrayList<>(needToSave), mongoTemplate);
+                    needToSave.clear();
+                }
+            }
+            if (CollectionUtils.isNotEmpty(needToSave)) {
+                saveLines(needToSave, mongoTemplate);
+            }
+        }
+        this.latch.countDown();
+        log.info("财付通-交易明细结束");
+    }
+}

src/main/java/ieven/server/webapp/service/tenpay/TenpayTradesScaleAnalysis.java

@@ -44,7 +44,7 @@ public class TenpayTradesScaleAnalysis extends Ops implements Runnable {
     log.info("财付通-账户透视开始");
     // 直接删除并创建表
     String newFileId = "";
-    String dateFileId = getFileId("财付通-交易明细", modelId, mongoTemplate);
+      String dateFileId = getFileId("财付通-交易明细", modelId, mongoTemplate);
     // 插入记录用户后续查询明细
     if (StringUtils.isNotBlank(dateFileId)) {
       MongoCursor<Document> cursor = getCursor(dateFileId, mongoTemplate);