Merge remote-tracking branch 'origin/master'

src/main/java/ieven/server/webapp/domain/alipay/HeaderProperties.java

@@ -24,6 +24,9 @@ public class HeaderProperties {
   private List<String> antiSecurityTradeEndAnalysisOutput;
   private List<String> orderedNames;
   private List<String>antiSecurityTrade;
+  private List<String>staticUserOutput;
+  private List<String>economicOrder;
+  private List<String>economicOrderOutput;
 
   public HeaderProperties() {}
 

src/main/java/ieven/server/webapp/service/FieldsService.java

@@ -75,6 +75,10 @@ public class FieldsService extends Ops {
     // 财付通-手机明细
     fields = matchHeaders(originFields, TenpayHeaders.getHeaders(4));
     this.updateLogicalFile(fields, "财付通-手机明细");
+
+      fields = matchHeaders(originFields, headerProperties.getEconomicOrder());
+      this.updateLogicalFile(fields, "经侦数据-订单明细");
+
   }
 
   void updateLogicalFile(List<Fields> fields, String matchType) {

src/main/java/ieven/server/webapp/service/alipay/AntiSecurityTrade3.java

@@ -52,7 +52,7 @@ public class AntiSecurityTrade3 extends Ops implements Runnable {
         String dateFileId3 = getFileId("经侦数据-订单明细", modelId, mongoTemplate);
 
         // 遍历登录日志全表
-        if (StringUtils.isNotBlank(dateFileId1) || StringUtils.isNotBlank(dateFileId2)) {
+        if (StringUtils.isNotBlank(dateFileId1) || StringUtils.isNotBlank(dateFileId2)|| StringUtils.isNotBlank(dateFileId3)) {
 
             MongoCursor<Document> cursor1 = getCursor(dateFileId1, mongoTemplate);
             if (cursor1.hasNext()) {
@@ -138,7 +138,7 @@ public class AntiSecurityTrade3 extends Ops implements Runnable {
             dataMap.put("交易时间", DataUtils.transferDateToStr(origin.getString("交易时间")));
 
             String jyzthm = String.valueOf(dataMap.get("交易主体户名"));
-            if (StringUtils.isBlank(jyzthm) || "-".equals(jyzthm)) {
+            if (StringUtils.isBlank(jyzthm) || "-".equals(jyzthm)||"null".equals(jyzthm)) {
                 String account = (String) dataMap.getOrDefault("交易主体账号", "");
                 if (StringUtils.isBlank(account) || "-".equals(account)) {
                 } else {
@@ -152,7 +152,7 @@ public class AntiSecurityTrade3 extends Ops implements Runnable {
             }
 
             String value = String.valueOf(dataMap.get("交易对手户名"));
-            if (StringUtils.isBlank(value) || "-".equals(value)) {
+            if (StringUtils.isBlank(value) || "-".equals(value)||"null".equals(value)) {
                 String account = (String) dataMap.getOrDefault("交易对手账号", "");
                 if (StringUtils.isBlank(account) || "-".equals(account)) {
                 } else {
@@ -220,7 +220,7 @@ public class AntiSecurityTrade3 extends Ops implements Runnable {
             }
             dataMap.put("交易时间", DataUtils.transferDateToStr(origin.getString("交易时间")));
             String jyzthm = String.valueOf(dataMap.get("交易主体户名"));
-            if (StringUtils.isBlank(jyzthm)) {
+            if (StringUtils.isBlank(jyzthm) || "-".equals(jyzthm)||"null".equals(jyzthm)) {
                 String account = (String) dataMap.getOrDefault("交易主体账号", "");
                 if (StringUtils.isBlank(account) || "-".equals(account)) {
                 } else {
@@ -233,7 +233,7 @@ public class AntiSecurityTrade3 extends Ops implements Runnable {
                 dataMap.put("交易主体户名", PublicStatic.specialFormatStr(jyzthm));
             }
             String value = String.valueOf(dataMap.get("交易对手户名"));
-            if (StringUtils.isBlank(value)) {
+            if (StringUtils.isBlank(value) || "-".equals(value)||"null".equals(value)) {
                 String account = (String) dataMap.getOrDefault("交易对手账号", "");
                 if (StringUtils.isBlank(account) || "-".equals(account)) {
                 } else {
@@ -265,7 +265,7 @@ public class AntiSecurityTrade3 extends Ops implements Runnable {
             dataMap.put("fileId", antiSecurityFileId);
             dataMap.put("交易主体账号", originDoc.getString("查询账号"));
             dataMap.put("支付订单号", originDoc.getString("支付机构内部订单号"));
-            String f1 = originDoc.getString("借贷类型");
+            String f1 = originDoc.getString("借贷标志");
             dataMap.put("交易金额", originDoc.getString("交易金额"));
             dataMap.put("交易余额", originDoc.getString("交易余额"));
             dataMap.put("交易类型", originDoc.getString("交易类型"));
@@ -293,7 +293,7 @@ public class AntiSecurityTrade3 extends Ops implements Runnable {
                 dataMap.put("交易对手户名", originDoc.getString("收款方的商户名称"));
             }
             String jyzthm = String.valueOf(dataMap.get("交易主体户名"));
-            if (StringUtils.isBlank(jyzthm)) {
+            if (StringUtils.isBlank(jyzthm) || "-".equals(jyzthm)||"null".equals(jyzthm)) {
                 String account = (String) dataMap.getOrDefault("交易主体账号", "");
                 if (StringUtils.isBlank(account) || "-".equals(account)) {
                 } else {
@@ -306,7 +306,7 @@ public class AntiSecurityTrade3 extends Ops implements Runnable {
                 dataMap.put("交易主体户名", PublicStatic.specialFormatStr(jyzthm));
             }
             String value = String.valueOf(dataMap.get("交易对手户名"));
-            if (StringUtils.isBlank(value)) {
+            if (StringUtils.isBlank(value) || "-".equals(value)||"null".equals(value)) {
                 String account = (String) dataMap.getOrDefault("交易对手账号", "");
                 if (StringUtils.isBlank(account) || "-".equals(account)) {
                 } else {

src/main/java/ieven/server/webapp/service/alipay/EconomicOrder.java

@@ -40,8 +40,8 @@ public class EconomicOrder extends Ops implements Runnable {
 
   @Override
   public void run() {
-    List<String> headers = headerProperties.getSecurityOrigin();
-    List<String> outputHeaders = headerProperties.getSecurityOutput();
+    List<String> headers = headerProperties.getEconomicOrder();
+    List<String> outputHeaders = headerProperties.getEconomicOrderOutput();
     List<Fields> matched = matchHeaders(originFields, headers);
     if (CollectionUtils.isNotEmpty(matched)) {
       String newFileId =

src/main/java/ieven/server/webapp/service/datastatic/DataStaticAnalysisTotal.java

@@ -51,13 +51,14 @@ public class DataStaticAnalysisTotal extends Ops implements Runnable {
         if (StringUtils.isNotBlank(dateFileId)) {
             MongoCursor<Document> cursor = getCursor(dateFileId, mongoTemplate);
             if (cursor.hasNext()) {
-                newFileId =createLogicAndDeleteOld("数据分析-主体信息汇总", modelId,headerProperties.getAntiSecurityTradeScaleAnalysisOutput(),mongoTemplate, fileService);
+                newFileId =createLogicAndDeleteOld("数据分析-主体信息汇总", modelId,headerProperties.getStaticUserOutput(),mongoTemplate, fileService);
                 insertFromTo(dateFileId, newFileId, modelId, mongoTemplate);
             }
             Set<String> alipayIds = new HashSet<>();
             Map<String, DataMap> idToData = new HashMap<>();
             Map<String, Set<String>> idToNames = new HashMap<>();
             Document origin;
+            List<DataMap> needToSave = new ArrayList<>(1000);
             while (cursor.hasNext()) {
                 try {
                     origin = cursor.next();
@@ -69,10 +70,11 @@ public class DataStaticAnalysisTotal extends Ops implements Runnable {
                         dataMap.put("fileId", newFileId);
                         dataMap.put("交易主体账号", alipayId);
                         dataMap.put("交易主体户名", "");
+                        getData(dataMap,origin);
                         idToData.put(alipayId, dataMap);
                     }
                     String alipayUsername = origin.getString("交易主体户名");
-                    if (StringUtils.isNotBlank(alipayUsername)) {
+                    if (StringUtils.isNotBlank(alipayUsername)&&!"null".equals(alipayUsername)) {
                         Set<String> namesSet;
                         if (idToNames.containsKey(alipayId)) {
                             namesSet = idToNames.get(alipayId);
@@ -95,9 +97,16 @@ public class DataStaticAnalysisTotal extends Ops implements Runnable {
                     if (CollectionUtils.isNotEmpty(namesSet)) {
                         dataMap.put("交易主体户名", StringUtils.join(namesSet, ";"));
                     }
-
+                    needToSave.add(dataMap);
+                    if (needToSave.size() >= 1000) {
+                        saveLines(new ArrayList<>(needToSave), mongoTemplate);
+                        needToSave.clear();
+                    }
                     //查询其他的基本信息
                 }
+                if (CollectionUtils.isNotEmpty(needToSave)) {
+                    saveLines(needToSave, mongoTemplate);
+                }
             }
         }
         latch.countDown();

src/main/resources/application.properties

@@ -42,5 +42,8 @@ headers.ordered-names=\u53CD\u8BC8\u6570\u636E-\u8BA2\u5355\u660E\u7EC6,\u6CBB\u
   \u4E94\u8054\u5355-\u6CE8\u518C\u4FE1\u606F,\u4E94\u8054\u5355-\u767B\u5F55\u65E5\u5FD7,\u4E94\u8054\u5355-\u4EA4\u6613\u8BB0\u5F55,\u4E94\u8054\u5355-\u8D26\u6237\u660E\u7EC6,\u4E94\u8054\u5355-\u8F6C\u8D26\u660E\u7EC6,\u4E94\u8054\u5355-\u8F6C\u8D26\u5206\u6790,\u4E94\u8054\u5355-\u8D26\u6237\u900F\u89C6,\u4E94\u8054\u5355-\u5BF9\u624B\u900F\u89C6,\u4E94\u8054\u5355-IP\u5730\u5740\u5206\u6790,\
   \u4E94\u8054\u5355-\u6536\u8D27\u5730\u5740\u5206\u6790
 headers.anti-security-trade=\u5e8f\u53f7,\u67e5\u8be2\u8d26\u53f7,\u652f\u4ed8\u673a\u6784\u5185\u90e8\u8ba2\u5355\u53f7,\u4ed8\u6b3e\u65b9\u652f\u4ed8\u5e10\u53f7,\u4ed8\u6b3e\u65b9\u94f6\u884c\u5361\u6240\u5c5e\u884c,\u4ed8\u6b3e\u65b9\u94f6\u884c\u5361\u53f7,\u4ea4\u6613\u91d1\u989d,\u4ea4\u6613\u4f59\u989d,\u6536\u6b3e\u65b9\u652f\u4ed8\u5e10\u53f7,\u6536\u6b3e\u65b9\u94f6\u884c\u5361\u6240\u5c5e\u884c,\u6536\u6b3e\u65b9\u94f6\u884c\u5361\u53f7,\u4ea4\u6613\u7c7b\u578b,\u4ea4\u6613\u65f6\u95f4,\u652f\u4ed8\u7c7b\u578b,\u501f\u8d37\u6807\u5fd7,\u5e01\u79cd,\u6d88\u8d39POS\u673a\u7f16\u53f7,\u6536\u6b3e\u65b9\u7684\u5546\u6237\u53f7,\u6536\u6b3e\u65b9\u7684\u5546\u6237\u540d\u79f0,\u4ea4\u6613\u8bbe\u5907\u53f7,\u94f6\u884c\u5916\u90e8\u6e20\u9053\u4ea4\u6613\u6d41\u6c34\u53f7,\u5907\u6ce8
+headers.static-user-output=\u4ea4\u6613\u4e3b\u4f53\u8d26\u53f7,\u4ea4\u6613\u4e3b\u4f53\u6237\u540d,\u8bc1\u4ef6\u53f7\u7801,\u7ed1\u5b9a\u94f6\u884c\u5361\u53f7,\u5f00\u6237\u94f6\u884c\u540d\u79f0,\u7ed1\u5b9a\u624b\u673a\u53f7,\u6ce8\u518c\u65f6\u95f4
+headers.economic-order=\u5e8f\u53f7,\u67e5\u8be2\u8d26\u53f7,\u652f\u4ed8\u673a\u6784\u5185\u90e8\u8ba2\u5355\u53f7,\u4ed8\u6b3e\u65b9\u652f\u4ed8\u5e10\u53f7,\u4ed8\u6b3e\u65b9\u94f6\u884c\u5361\u6240\u5c5e\u884c,\u4ed8\u6b3e\u65b9\u94f6\u884c\u5361\u53f7,\u4ea4\u6613\u91d1\u989d,\u4ea4\u6613\u4f59\u989d,\u6536\u6b3e\u65b9\u652f\u4ed8\u5e10\u53f7,\u6536\u6b3e\u65b9\u94f6\u884c\u5361\u6240\u5c5e\u884c,\u6536\u6b3e\u65b9\u94f6\u884c\u5361\u53f7,\u4ea4\u6613\u7c7b\u578b,\u4ea4\u6613\u65f6\u95f4,\u652f\u4ed8\u7c7b\u578b,\u501f\u8d37\u6807\u5fd7,\u5e01\u79cd,\u6d88\u8d39POS\u673a\u7f16\u53f7,\u6536\u6b3e\u65b9\u7684\u5546\u6237\u53f7,\u6536\u6b3e\u65b9\u7684\u5546\u6237\u540d\u79f0,\u4ea4\u6613\u8bbe\u5907\u53f7,\u94f6\u884c\u5916\u90e8\u6e20\u9053\u4ea4\u6613\u6d41\u6c34\u53f7,\u5907\u6ce8
+headers.economic-order-output=\u5e8f\u53f7,\u67e5\u8be2\u8d26\u53f7,\u652f\u4ed8\u673a\u6784\u5185\u90e8\u8ba2\u5355\u53f7,\u4ed8\u6b3e\u65b9\u652f\u4ed8\u5e10\u53f7,\u4ed8\u6b3e\u65b9\u94f6\u884c\u5361\u6240\u5c5e\u884c,\u4ed8\u6b3e\u65b9\u94f6\u884c\u5361\u53f7,\u4ea4\u6613\u91d1\u989d,\u4ea4\u6613\u4f59\u989d,\u6536\u6b3e\u65b9\u652f\u4ed8\u5e10\u53f7,\u6536\u6b3e\u65b9\u94f6\u884c\u5361\u6240\u5c5e\u884c,\u6536\u6b3e\u65b9\u94f6\u884c\u5361\u53f7,\u4ea4\u6613\u7c7b\u578b,\u4ea4\u6613\u65f6\u95f4,\u652f\u4ed8\u7c7b\u578b,\u501f\u8d37\u6807\u5fd7,\u5e01\u79cd,\u6d88\u8d39POS\u673a\u7f16\u53f7,\u6536\u6b3e\u65b9\u7684\u5546\u6237\u53f7,\u6536\u6b3e\u65b9\u7684\u5546\u6237\u540d\u79f0,\u4ea4\u6613\u8bbe\u5907\u53f7,\u94f6\u884c\u5916\u90e8\u6e20\u9053\u4ea4\u6613\u6d41\u6c34\u53f7,\u5907\u6ce8
 #\u8F6C\u8D26\u5206\u6790-\u603B\u91D1\u989Ddesc
 #\u8D26\u6237\u900F\u89C6-\u4EA4\u6613\u603B\u989Ddesc