增加网黑平台

src/main/java/ieven/server/webapp/domain/alipay/HeaderProperties.java

@@ -34,6 +34,9 @@ public class HeaderProperties {
   private List<String> securityOriginNew;
   private List<String> securityOutputNew;
 
+  private List<String>blackOrder;
+  private List<String>blackOrderOutput;
+
   public HeaderProperties() {}
 
 }

src/main/java/ieven/server/webapp/domain/file/FileService.java

@@ -55,7 +55,7 @@ import java.util.stream.Collectors;
 @EnableAsync
 @Slf4j
 public class FileService {
-    String[] order1 = {"反诈数据-订单明细", "治安数据-订单明细","经侦数据-订单明细", "反诈治安-交易明细", "反诈治安-账户透视", "反诈治安-对手透视"};
+    String[] order1 = {"反诈数据-订单明细", "治安数据-订单明细","经侦数据-订单明细","网黑数据-订单明细", "反诈治安-交易明细", "反诈治安-账户透视", "反诈治安-对手透视"};
     String[] order2 = {
             "五联单-注册信息",
             "五联单-登录日志",

src/main/java/ieven/server/webapp/service/FieldsService.java

@@ -81,6 +81,9 @@ public class FieldsService extends Ops {
         fields = matchHeaders(originFields, headerProperties.getEconomicOrder());
         this.updateLogicalFile(fields, "经侦数据-订单明细");
 
+        fields = matchHeaders(originFields, headerProperties.getBlackOrder());
+        this.updateLogicalFile(fields, "网黑数据-订单明细");
+
     }
 
     void updateLogicalFile(List<Fields> fields, String matchType) {

src/main/java/ieven/server/webapp/service/alipay/AlipayService.java

@@ -70,7 +70,7 @@ public class AlipayService {
         // 无限制大小的threadpool
         ExecutorService exec = Executors.newCachedThreadPool();
         log.info("latch");
-        CountDownLatch latch = new CountDownLatch(8);
+        CountDownLatch latch = new CountDownLatch(9);
         // 五联单-注册信息
         exec.execute(new ALIPAY_ORIGIN_REGISTER(originFields, modelId, mongoTemplate, latch));
         // 五联单-登录日志
@@ -88,9 +88,9 @@ public class AlipayService {
         exec.execute(
                 new SecurityOrder(
                         headerProperties, originFields, modelId, mongoTemplate, fileService, latch));
-        // 经侦数据-订单数据
+        // 网黑数据-订单数据
         exec.execute(
-                new EconomicOrder(
+                new BlackOrder(
                         headerProperties, originFields, modelId, mongoTemplate, fileService, latch));
         latch.await();
         log.info("latch");

src/main/java/ieven/server/webapp/service/alipay/AntiSecurityTrade3.java

@@ -54,6 +54,7 @@ public class AntiSecurityTrade3 extends Ops implements Runnable {
         String dateFileId1 = getFileId("反诈数据-订单明细", modelId, mongoTemplate);
         String dateFileId2 = getFileId("治安数据-订单明细", modelId, mongoTemplate);
         String dateFileId3 = getFileId("经侦数据-订单明细", modelId, mongoTemplate);
+        String dateFileId4 = getFileId("网黑数据-订单明细", modelId, mongoTemplate);
 
         if(StringUtils.isNotBlank(dateFileId1)){
             MongoCursor<Document> cursor1 = getCursor(dateFileId1, mongoTemplate);
@@ -76,6 +77,13 @@ public class AntiSecurityTrade3 extends Ops implements Runnable {
                 insertData3(cursor3, antiSecurityFileId);
             }
         }
+        if(StringUtils.isNotBlank(dateFileId4)){
+            MongoCursor<Document> cursor4 = getCursor(dateFileId4, mongoTemplate);
+            if (cursor4.hasNext()) {
+                antiSecurityFileId = StringUtils.isBlank(antiSecurityFileId)?antiSecurityFileId():antiSecurityFileId;
+                insertData4(cursor4, antiSecurityFileId);
+            }
+        }
         latch.countDown();
     }
 
@@ -393,4 +401,101 @@ public class AntiSecurityTrade3 extends Ops implements Runnable {
             saveLines(needToSave, mongoTemplate);
         }
     }
+
+    public void insertData4(MongoCursor<Document> cursor, String antiSecurityFileId) {
+        Document origin;
+        List<DataMap> needToSave = new ArrayList<>(1000);
+        Map<String, Object> dataMapUser = new HashMap<>();
+        while (cursor.hasNext()) {
+            origin = cursor.next();
+            DataMap dataMap = new DataMap();
+            dataMap.put("fileId", antiSecurityFileId);
+            dataMap.put("支付订单号", origin.getString("支付订单号"));
+            dataMap.put("交易流水号", origin.getString("交易流水号"));
+            String flag = origin.getString("交易主体的出入账标识");
+            dataMap.put("交易主体的出入账标识", flag);
+            dataMap.put("交易类型", origin.getString("交易类型"));
+            dataMap.put("支付类型", origin.getString("支付类型"));
+            dataMap.put("币种", origin.getString("币种"));
+            dataMap.put("交易金额", origin.getString("交易金额"));
+            dataMap.put("交易余额", origin.getString("交易余额"));
+            dataMap.put("消费POS机编号", origin.getString("消费POS机编号"));
+            dataMap.put("交易设备类型", origin.getString("交易设备类型"));
+            dataMap.put("交易支付设备IP", origin.getString("交易支付设备IP"));
+            dataMap.put("MAC地址", origin.getString("MAC地址"));
+            dataMap.put("交易地点经度", origin.getString("交易地点经度"));
+            dataMap.put("交易地点纬度", origin.getString("交易地点纬度"));
+            dataMap.put("备注", origin.getString("备注"));
+            if ("出账".equals(flag)) {
+                dataMap.put("交易主体账号", origin.getString("付款支付账号"));
+                dataMap.put("交易主体银行卡号", origin.getString("付款银行卡号"));
+                dataMap.put("交易主体银行名称", origin.getString("付款银行卡银行名称"));
+                dataMap.put("交易主体户名", "");
+                dataMap.put("交易对手账号", origin.getString("收款支付账号"));
+                dataMap.put("交易对手银行卡号", origin.getString("收款银行卡号"));
+                dataMap.put("交易对手银行名称", origin.getString("收款银行卡银行名称"));
+                dataMap.put("交易对手户名", origin.getString("收款方的商户名称"));
+                String jyztzh = String.valueOf(origin.getString("收款支付账号"));
+                String jyztzhhm = String.valueOf(origin.getString("收款方的商户名称"));
+                if (StringUtils.isNotBlank(jyztzh) && !"null".equals(jyztzh) && StringUtils.isNotBlank(jyztzhhm) && !"null".equals(jyztzhhm)) {
+                    dataMapUser.put(jyztzh, jyztzhhm);
+                }
+
+            } else if ("入账".equals(flag)) {
+                dataMap.put("交易主体账号", origin.getString("收款支付账号"));
+                dataMap.put("交易主体银行卡号", origin.getString("收款银行卡号"));
+                dataMap.put("交易主体银行名称", origin.getString("收款银行卡银行名称"));
+                dataMap.put("交易主体户名", origin.getString("收款方的商户名称"));
+                dataMap.put("交易对手账号", origin.getString("付款支付账号"));
+                dataMap.put("交易对手银行卡号", origin.getString("付款银行卡号"));
+                dataMap.put("交易对手银行名称", origin.getString("付款银行卡银行名称"));
+                dataMap.put("交易对手户名", "");
+                String jyztzh = String.valueOf(origin.getString("收款支付账号"));
+                String jyztzhhm = String.valueOf(origin.getString("收款方的商户名称"));
+                if (StringUtils.isNotBlank(jyztzh) && !"null".equals(jyztzh) && StringUtils.isNotBlank(jyztzhhm) && !"null".equals(jyztzhhm)) {
+                    dataMapUser.put(jyztzh, jyztzhhm);
+                }
+            }
+            dataMap.put("交易时间", DataUtils.transferDateToStr(origin.getString("交易时间")));
+
+            String jyzthm = String.valueOf(dataMap.get("交易主体户名"));
+            if (StringUtils.isBlank(jyzthm) || "-".equals(jyzthm) || "null".equals(jyzthm)) {
+                String account = (String) dataMap.getOrDefault("交易主体账号", "");
+                if (StringUtils.isBlank(account) || "-".equals(account)) {
+                } else {
+                    String newName = accountToName.getOrDefault(account, "");
+                    if (StringUtils.isNotBlank(newName)) {
+                        dataMap.put("交易主体户名", PublicStatic.specialFormatStr(newName));
+                    } else {
+                        dataMap.put("交易主体户名", PublicStatic.specialFormatStr(String.valueOf(dataMapUser.get(account))));
+                    }
+                }
+            } else {
+                dataMap.put("交易主体户名", PublicStatic.specialFormatStr(jyzthm));
+            }
+            String value = String.valueOf(dataMap.get("交易对手户名"));
+            if (StringUtils.isBlank(value) || "-".equals(value) || "null".equals(value)) {
+                String account = (String) dataMap.getOrDefault("交易对手账号", "");
+                if (StringUtils.isBlank(account) || "-".equals(account)) {
+                } else {
+                    String newName = accountToName.getOrDefault(account, "");
+                    if (StringUtils.isNotBlank(newName)) {
+                        dataMap.put("交易对手户名", PublicStatic.specialFormatStr(newName));
+                    } else {
+                        dataMap.put("交易对手户名", PublicStatic.specialFormatStr(String.valueOf(dataMapUser.get(account))));
+                    }
+                }
+            } else {
+                dataMap.put("交易对手户名", PublicStatic.specialFormatStr(value));
+            }
+            needToSave.add(dataMap);
+            if (needToSave.size() >= 1000) {
+                saveLines(new ArrayList<>(needToSave), mongoTemplate);
+                needToSave.clear();
+            }
+        }
+        if (CollectionUtils.isNotEmpty(needToSave)) {
+            saveLines(needToSave, mongoTemplate);
+        }
+    }
 }

src/main/java/ieven/server/webapp/service/alipay/BlackOrder.java

@@ -0,0 +1,71 @@
+package ieven.server.webapp.service.alipay;
+
+import com.mongodb.client.MongoCursor;
+import ieven.server.webapp.domain.alipay.HeaderProperties;
+import ieven.server.webapp.domain.data.DataMap;
+import ieven.server.webapp.domain.data.Fields;
+import ieven.server.webapp.domain.file.FileService;
+import ieven.server.webapp.service.Ops;
+import org.apache.commons.collections4.CollectionUtils;
+import org.bson.Document;
+import org.springframework.data.mongodb.core.MongoTemplate;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+
+public class BlackOrder extends Ops implements Runnable {
+
+  private HeaderProperties headerProperties;
+  List<Fields> originFields;
+  String modelId;
+  MongoTemplate mongoTemplate;
+  FileService fileService;
+  CountDownLatch latch;
+
+  public BlackOrder(
+      HeaderProperties headerProperties,
+      List<Fields> originFields,
+      String modelId,
+      MongoTemplate mongoTemplate,
+      FileService fileService,
+      CountDownLatch latch) {
+    this.headerProperties = headerProperties;
+    this.originFields = originFields;
+    this.modelId = modelId;
+    this.mongoTemplate = mongoTemplate;
+    this.fileService = fileService;
+    this.latch = latch;
+  }
+
+  @Override
+  public void run() {
+    List<String> headers = headerProperties.getBlackOrder();
+    List<String> outputHeaders = headerProperties.getBlackOrderOutput();
+    List<Fields> matched = matchHeaders(originFields, headers);
+    if (CollectionUtils.isNotEmpty(matched)) {
+      String newFileId =
+          createLogicAndDeleteOld("网黑数据-订单明细", modelId, outputHeaders, mongoTemplate, fileService);
+      for (Fields fields : matched) {
+        String fileId = fields.getFileId();
+        MongoCursor<Document> cursor = getCursor(fileId, mongoTemplate);
+        Document originDoc;
+        List<DataMap> needToSave = new ArrayList<>(1000);
+        while (cursor.hasNext()) {
+          originDoc = cursor.next();
+          // 拷贝到新的
+          DataMap newDoc = initWithOrigin(originDoc, newFileId);
+          needToSave.add(newDoc);
+          if (needToSave.size() >= 1000) {
+            saveLines(new ArrayList<>(needToSave), mongoTemplate);
+            needToSave.clear();
+          }
+        }
+        if (CollectionUtils.isNotEmpty(needToSave)) {
+          saveLines(needToSave, mongoTemplate);
+        }
+      }
+    }
+    latch.countDown();
+  }
+}

src/main/resources/application.properties

@@ -51,5 +51,7 @@ headers.economic-order=\u5e8f\u53f7,\u67e5\u8be2\u8d26\u53f7,\u652f\u4ed8\u673a\
 headers.economic-order-output=\u5e8f\u53f7,\u67e5\u8be2\u8d26\u53f7,\u652f\u4ed8\u673a\u6784\u5185\u90e8\u8ba2\u5355\u53f7,\u4ed8\u6b3e\u65b9\u652f\u4ed8\u5e10\u53f7,\u4ed8\u6b3e\u65b9\u94f6\u884c\u5361\u6240\u5c5e\u884c,\u4ed8\u6b3e\u65b9\u94f6\u884c\u5361\u53f7,\u4ea4\u6613\u91d1\u989d,\u4ea4\u6613\u4f59\u989d,\u6536\u6b3e\u65b9\u652f\u4ed8\u5e10\u53f7,\u6536\u6b3e\u65b9\u94f6\u884c\u5361\u6240\u5c5e\u884c,\u6536\u6b3e\u65b9\u94f6\u884c\u5361\u53f7,\u4ea4\u6613\u7c7b\u578b,\u4ea4\u6613\u65f6\u95f4,\u652f\u4ed8\u7c7b\u578b,\u501f\u8d37\u6807\u5fd7,\u5e01\u79cd,\u6d88\u8d39POS\u673a\u7f16\u53f7,\u6536\u6b3e\u65b9\u7684\u5546\u6237\u53f7,\u6536\u6b3e\u65b9\u7684\u5546\u6237\u540d\u79f0,\u4ea4\u6613\u8bbe\u5907\u53f7,\u94f6\u884c\u5916\u90e8\u6e20\u9053\u4ea4\u6613\u6d41\u6c34\u53f7,\u5907\u6ce8
 headers.static-user-all-output=\u8bc1\u4ef6\u53f7,\u8d26\u6237\u540d\u79f0,\u8d26\u53f7
 headers.user-account-output=\u652f\u4ed8\u8ba2\u5355\u53f7,\u4ea4\u6613\u6d41\u6c34\u53f7,\u4ea4\u6613\u4e3b\u4f53\u94f6\u884c\u540d\u79f0,\u4ea4\u6613\u4e3b\u4f53\u8d26\u53f7,\u4ea4\u6613\u4e3b\u4f53\u94f6\u884c\u5361\u53f7,\u4ea4\u6613\u4e3b\u4f53\u6237\u540d,\u4ea4\u6613\u7c7b\u578b,\u652f\u4ed8\u7c7b\u578b,\u501f\u8d37\u7c7b\u578b,\u4ea4\u6613\u65f6\u95f4,\u5e01\u79cd,\u4ea4\u6613\u91d1\u989d,\u4ea4\u6613\u4f59\u989d,\u4ea4\u6613\u5bf9\u624b\u8d26\u53f7,\u4ea4\u6613\u5bf9\u624b\u94f6\u884c\u5361\u53f7,\u4ea4\u6613\u5bf9\u624b\u94f6\u884c\u540d\u79f0,\u4ea4\u6613\u5bf9\u624b\u6237\u540d,\u6d88\u8d39POS\u673a\u7f16\u53f7,\u4ea4\u6613\u8bbe\u5907\u7c7b\u578b,\u4ea4\u6613\u652f\u4ed8\u8bbe\u5907IP,MAC\u5730\u5740,\u4ea4\u6613\u5730\u70b9\u7ecf\u5ea6,\u4ea4\u6613\u5730\u70b9\u7eac\u5ea6,\u6d88\u8d39\u540d\u79f0,\u4ea4\u6613\u6765\u6e90\u5730,\u4ea4\u6613\u72b6\u6001,\u5907\u6ce8,\u7c7b\u578b
+headers.black-order=\u5e8f\u53f7,\u67e5\u8be2\u8d26\u53f7,\u4ea4\u6613\u6d41\u6c34\u53f7,\u4ed8\u6b3e\u652f\u4ed8\u5e10\u53f7,\u4ed8\u6b3e\u94f6\u884c\u5361\u94f6\u884c\u540d\u79f0,\u4ed8\u6b3e\u94f6\u884c\u5361\u53f7,\u4ea4\u6613\u91d1\u989d,\u4ea4\u6613\u4f59\u989d,\u6536\u6b3e\u652f\u4ed8\u5e10\u53f7,\u6536\u6b3e\u94f6\u884c\u5361\u94f6\u884c\u540d\u79f0,\u6536\u6b3e\u94f6\u884c\u5361\u53f7,\u4ea4\u6613\u7c7b\u578b,\u4ea4\u6613\u65f6\u95f4,\u652f\u4ed8\u7c7b\u578b,\u4ea4\u6613\u4e3b\u4f53\u7684\u51fa\u5165\u8d26\u6807\u8bc6,\u5e01\u79cd,\u6d88\u8d39POS\u673a\u7f16\u53f7,\u6536\u6b3e\u65b9\u7684\u5546\u6237\u53f7,\u4ea4\u6613\u8bbe\u5907\u7c7b\u578b,\u4ea4\u6613\u652f\u4ed8\u8bbe\u5907IP,MAC\u5730\u5740,\u4ea4\u6613\u5730\u70b9\u7ecf\u5ea6,\u4ea4\u6613\u5730\u70b9\u7eac\u5ea6,\u4ea4\u6613\u8bbe\u5907\u53f7,\u94f6\u884c\u5916\u90e8\u6e20\u9053\u4ea4\u6613\u6d41\u6c34\u53f7,\u6536\u6b3e\u5546\u6237\u540d\u79f0,\u5907\u6ce8
+headers.black-order-output=\u67e5\u8be2\u8d26\u53f7,\u4ea4\u6613\u6d41\u6c34\u53f7,\u4ed8\u6b3e\u652f\u4ed8\u5e10\u53f7,\u4ed8\u6b3e\u94f6\u884c\u5361\u94f6\u884c\u540d\u79f0,\u4ed8\u6b3e\u94f6\u884c\u5361\u53f7,\u4ea4\u6613\u91d1\u989d,\u4ea4\u6613\u4f59\u989d,\u6536\u6b3e\u652f\u4ed8\u5e10\u53f7,\u6536\u6b3e\u94f6\u884c\u5361\u94f6\u884c\u540d\u79f0,\u6536\u6b3e\u94f6\u884c\u5361\u53f7,\u4ea4\u6613\u7c7b\u578b,\u4ea4\u6613\u65f6\u95f4,\u652f\u4ed8\u7c7b\u578b,\u4ea4\u6613\u4e3b\u4f53\u7684\u51fa\u5165\u8d26\u6807\u8bc6,\u5e01\u79cd,\u6d88\u8d39POS\u673a\u7f16\u53f7,\u6536\u6b3e\u65b9\u7684\u5546\u6237\u53f7,\u4ea4\u6613\u8bbe\u5907\u7c7b\u578b,\u4ea4\u6613\u652f\u4ed8\u8bbe\u5907IP,MAC\u5730\u5740,\u4ea4\u6613\u5730\u70b9\u7ecf\u5ea6,\u4ea4\u6613\u5730\u70b9\u7eac\u5ea6,\u4ea4\u6613\u8bbe\u5907\u53f7,\u94f6\u884c\u5916\u90e8\u6e20\u9053\u4ea4\u6613\u6d41\u6c34\u53f7,\u6536\u6b3e\u5546\u6237\u540d\u79f0,\u5907\u6ce8
 #\u8F6C\u8D26\u5206\u6790-\u603B\u91D1\u989Ddesc
 #\u8D26\u6237\u900F\u89C6-\u4EA4\u6613\u603B\u989Ddesc