提交新的修改

src/main/java/ieven/server/webapp/api/AlipayController.java

@@ -9,9 +9,6 @@ import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
 
-import java.util.LinkedHashMap;
-import java.util.Map;
-
 @RestController
 @RequestMapping("/alipay")
 public class AlipayController {
@@ -23,6 +20,10 @@ public class AlipayController {
      */
     @PostMapping("/extractPersonInfo")
     public Mapped extractPersonInfo(@RequestBody ModelIdInput modelIdInput) {
-        return alipayService.extractPersonInfo(modelIdInput);
+        String modelId = modelIdInput.getModelId();
+        if (modelId == null) {
+            return Mapped.ERROR("没有选择模型!");
+        }
+        return alipayService.extractAll(modelId);
     }
 }

src/main/java/ieven/server/webapp/api/DataController.java

@@ -2,12 +2,11 @@ package ieven.server.webapp.api;
 
 import ieven.server.webapp.domain.FileIdInput;
 import ieven.server.webapp.domain.IdInput;
-import ieven.server.webapp.domain.data.DataMap;
-import ieven.server.webapp.domain.data.DataRepository;
-import ieven.server.webapp.domain.data.DataService;
-import ieven.server.webapp.domain.data.ViewInput;
+import ieven.server.webapp.domain.data.*;
 import ieven.server.webapp.infrastructure.wrapper.Mapped;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.poi.util.StringUtil;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
@@ -30,4 +29,12 @@ public class DataController {
     public Mapped viewHeader(@RequestBody FileIdInput idInput) {
         return dataService.viewHeaders(idInput);
     }
+
+    @PostMapping("/viewDataByQuery")
+    public Mapped viewDataByQuery(@RequestBody QueryParam param) {
+        if (StringUtils.isEmpty(param.getFileId())) {
+            return Mapped.ERROR("文件选择为空!");
+        }
+        return dataService.viewDataByQueryParam(param);
+    }
 }

src/main/java/ieven/server/webapp/domain/alipay/AlipayHeaderOps.java

@@ -0,0 +1,51 @@
+package ieven.server.webapp.domain.alipay;
+
+import ieven.server.webapp.domain.alipay.AlipayHeaders;
+import ieven.server.webapp.domain.data.Fields;
+import ieven.server.webapp.domain.file.FileStatus;
+import ieven.server.webapp.domain.file.LogicalFile;
+import ieven.server.webapp.infrastructure.SpringContextUtil;
+import ieven.server.webapp.util.DataUtils;
+import org.springframework.data.mongodb.core.MongoTemplate;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+
+public class AlipayHeaderOps {
+
+    /**
+     * 创建文件和表头并返回表头
+     * @param type 类型id
+     * @param modelId 模型id
+     * @return
+     */
+    public static Fields createFileAndFields(int type, String modelId) {
+        List<String> headerNames = AlipayHeaders.getHeaders(type);
+        String filename = AlipayHeaders.getGeneratedName(type);
+        LogicalFile logicalFile = new LogicalFile();
+        logicalFile.setFilename(filename);
+        logicalFile.setStatus(FileStatus.STATUS_LOADING);
+        String current = DataUtils.currentDate();
+        logicalFile.setUploadDate(current);
+        logicalFile.setModelId(modelId);
+        logicalFile.setGenerated(Boolean.TRUE);
+        MongoTemplate mongoTemplate = SpringContextUtil.getBean(MongoTemplate.class);
+        //插入文件获取文件id
+        logicalFile = mongoTemplate.insert(logicalFile);
+        Fields fields = new Fields();
+        fields.setFileId(logicalFile.getId());
+        //插入表头
+        LinkedHashMap<String, String> header = new LinkedHashMap<>();
+        LinkedHashMap<String, String> headerReversed = new LinkedHashMap<>();
+        for (int i = 0; i < headerNames.size(); i++) {
+            String field = headerNames.get(i);
+            String alias = "f" + i;
+            header.put(alias, field);
+            headerReversed.put(field, alias);
+        }
+        fields.setFields(header);
+        fields.setFieldsReverse(headerReversed);
+        return mongoTemplate.insert(fields);
+    }
+}

src/main/java/ieven/server/webapp/domain/alipay/AlipayHeaders.java

@@ -0,0 +1,69 @@
+package ieven.server.webapp.domain.alipay;
+
+import ieven.server.webapp.util.DataUtils;
+
+import java.util.*;
+
+/**
+ * 固定的表头
+ */
+public class AlipayHeaders {
+    private static final Map<Integer, List<String>> mappedHeaders = new HashMap<>();
+    private static final Map<Integer, String> mappedNames = new HashMap<>();
+    public static final String REGISTER_1 = "卡类型";
+    public static final String REGISTER_2 = "银行";
+    public static final String REGISTER_3 = "银行卡号";
+    public static final String LOGIN_1 = "日期";
+    public static final String LOGIN_2 = "时间";
+    public static final String TRADE_1 = "买家拆分1";
+    public static final String TRADE_2 = "买家拆分2";
+    public static final String TRADE_3 = "买家拆分3";
+    public static final String TRADE_4 = "卖家拆分1";
+    public static final String TRADE_5 = "卖家拆分2";
+    public static final String TRADE_6 = "卖家拆分3";
+
+    static {
+        mappedHeaders.put(AlipayType.ALIPAY_ORIGIN_REGISTER,
+                Arrays.asList("用户ID", "登录邮箱", "登录手机", "账户名称", "证件类型", "证件号", "可用余额", "绑定手机", "绑定银行卡", "对应的协查数据"));
+        mappedHeaders.put(AlipayType.ALIPAY_GENERATED_REGISTER,
+                Arrays.asList("用户ID", "登录邮箱", "登录手机", "账户名称", "证件类型", "证件号", "可用余额", "绑定手机", "绑定银行卡", "对应的协查数据",
+                        REGISTER_1, REGISTER_2, REGISTER_3));
+        mappedHeaders.put(AlipayType.ALIPAY_ORIGIN_LOGIN,
+                Arrays.asList("登陆账号", "支付宝用户ID", "账户名", "客户端ip", "操作发生时间", "对应的协查数据"));
+        mappedHeaders.put(AlipayType.ALIPAY_GENERATED_LOGIN,
+                Arrays.asList("登陆账号", "支付宝用户ID", "账户名", "客户端ip", "操作发生时间", "对应的协查数据", LOGIN_1, LOGIN_2));
+        mappedHeaders.put(AlipayType.ALIPAY_ORIGIN_TRADE,
+                Arrays.asList("交易号", "外部交易号", "交易状态", "合作伙伴ID", "买家用户id", "买家信息", "卖家用户id", "卖家信息", "交易金额(元)", "收款时间",
+                        "最后修改时间", "创建时间", "交易类型", "来源地", "商品名称", "收货人地址", "对应的协查数据"));
+        mappedHeaders.put(AlipayType.ALIPAY_GENERATED_TRADE,
+                Arrays.asList("交易号", "外部交易号", "交易状态", "合作伙伴ID", "买家用户id", "买家信息", "卖家用户id", "卖家信息", "交易金额(元)", "收款时间",
+                        "最后修改时间", "创建时间", "交易类型", "来源地", "商品名称", "收货人地址", "对应的协查数据",
+                        AlipayHeaders.TRADE_1,AlipayHeaders.TRADE_2,AlipayHeaders.TRADE_3,AlipayHeaders.TRADE_4,AlipayHeaders.TRADE_5,AlipayHeaders.TRADE_6));
+        mappedHeaders.put(AlipayType.ALIPAY_ORIGIN_ACCOUNT,
+                Arrays.asList("交易号", "商户订单号", "交易创建时间", "付款时间", "最近修改时间", "交易来源地", "类型", "用户信息", "交易对方信息", "消费名称", "金额(元)", "收/支", "交易状态", "备注", "对应的协查数据"));
+        mappedHeaders.put(AlipayType.ALIPAY_GENERATED_ACCOUNT,
+                Arrays.asList("交易号", "商户订单号", "交易创建时间", "付款时间", "最近修改时间", "交易来源地", "类型", "用户信息", "交易对方信息", "消费名称", "金额(元)", "收/支", "交易状态", "备注", "对应的协查数据"));
+        mappedHeaders.put(AlipayType.ALIPAY_ORIGIN_TRANSFER,
+                Arrays.asList("交易号", "付款方支付宝账号", "收款方支付宝账号", "收款机构信息", "到账时间", "转账金额(元)", "转账产品名称", "交易发生地", "提现流水号", "对应的协查数据"));
+        mappedHeaders.put(AlipayType.ALIPAY_GENERATED_TRANSFER,
+                Arrays.asList("交易号", "付款方支付宝账号", "收款方支付宝账号", "收款机构信息", "到账时间", "转账金额(元)", "转账产品名称", "交易发生地", "提现流水号", "对应的协查数据"));
+
+    }
+
+    //文件名
+    static {
+        mappedNames.put(AlipayType.ALIPAY_GENERATED_REGISTER, "注册信息");
+        mappedNames.put(AlipayType.ALIPAY_GENERATED_LOGIN, "登录日志");
+        mappedNames.put(AlipayType.ALIPAY_GENERATED_TRADE, "交易记录");
+        mappedNames.put(AlipayType.ALIPAY_GENERATED_ACCOUNT, "账户明细");
+        mappedNames.put(AlipayType.ALIPAY_GENERATED_TRANSFER, "转账明细");
+    }
+
+    public static List<String> getHeaders(int type) {
+        return mappedHeaders.get(type);
+    }
+
+    public static String getGeneratedName(int type) {
+        return mappedNames.get(type) + " " + DataUtils.currentDate();
+    }
+}

src/main/java/ieven/server/webapp/domain/alipay/AlipayService.java

@@ -4,14 +4,12 @@ import com.mongodb.client.MongoCursor;
 import ieven.server.webapp.domain.ModelIdInput;
 import ieven.server.webapp.domain.data.DataMap;
 import ieven.server.webapp.domain.data.Fields;
-import ieven.server.webapp.domain.data.FieldsService;
 import ieven.server.webapp.domain.file.FileStatus;
 import ieven.server.webapp.domain.file.LogicalFile;
 import ieven.server.webapp.infrastructure.wrapper.Mapped;
 import ieven.server.webapp.util.DataUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.ObjectUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.bson.Document;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -30,137 +28,300 @@ public class AlipayService {
     @Autowired
     private MongoTemplate mongoTemplate;
     @Autowired
-    private FieldsService fieldsService;
-    @Autowired
     @Lazy
     private AlipayService alipayService;
-    private final static String FNAME_PERSONINFO = "提取的人员信息表";
-    private final static String FIELDNAME_BANKCARD = "绑定银行卡";
-    private final static String EXTRACT_BANKCARDTYPE = "提取银行卡类型";
-    private final static String EXTRACT_BANKCARDCODE = "提取银行卡开户行";
-    private final static String EXTRACT_BANKCARD = "提取银行卡号";
-    private final static List<String> PERSONINFO_HEADERNAMES = Arrays.asList("用户ID", "登录邮箱", "登录手机", "账户名称", "证件类型", "证件号", "绑定手机", "绑定银行卡", "提取银行卡类型", "提取银行卡开户行", "提取银行卡号");
-
-    public Mapped testCursor() {
-        Query query = new Query(Criteria.where("fileId").is("618ce033a14aa6502ed010ba"));
-        MongoCursor<Document> cursor = mongoTemplate.getCollection("data")
-                .find(query.getQueryObject())
-                .batchSize(1000)
-                .noCursorTimeout(true)
-                .cursor();
-        Document doc;
-        while (cursor.hasNext()) {
-            //写法1（建议）
-            doc = cursor.next();
-            log.info(doc.getString("f4"));
+
+    public Mapped extractAll(String modelId) {
+        extractPersonInfo(modelId);
+        //登录日志
+        Fields outputLogin = AlipayHeaderOps.createFileAndFields(AlipayType.ALIPAY_GENERATED_LOGIN, modelId);
+        List<Fields> loginFieldsList = checkFields(modelId, AlipayType.ALIPAY_ORIGIN_LOGIN);
+        for (Fields origin : loginFieldsList) {
+            alipayService.extractLogin(getCursor(origin.getFileId()), origin, outputLogin);
+        }
+        //交易记录
+        Fields outputTrade = AlipayHeaderOps.createFileAndFields(AlipayType.ALIPAY_GENERATED_TRADE, modelId);
+        List<Fields> tradeFieldsList = checkFields(modelId, AlipayType.ALIPAY_ORIGIN_TRADE);
+        for (Fields origin : tradeFieldsList) {
+            alipayService.extractTrade(getCursor(origin.getFileId()), origin, outputTrade);
+        }
+        //账户明细
+        Fields outputAccount = AlipayHeaderOps.createFileAndFields(AlipayType.ALIPAY_GENERATED_ACCOUNT, modelId);
+        List<Fields> accountFieldsList = checkFields(modelId, AlipayType.ALIPAY_ORIGIN_ACCOUNT);
+        for (Fields origin : accountFieldsList) {
+            alipayService.extractAccount(getCursor(origin.getFileId()), origin, outputAccount);
+        }
+        //转账明细
+        Fields outputTransfer = AlipayHeaderOps.createFileAndFields(AlipayType.ALIPAY_GENERATED_TRANSFER, modelId);
+        List<Fields> transferFieldsList = checkFields(modelId, AlipayType.ALIPAY_ORIGIN_TRANSFER);
+        for (Fields origin : transferFieldsList) {
+            alipayService.extractTransfer(getCursor(origin.getFileId()), origin, outputTransfer);
         }
-        return null;
+        return Mapped.OK();
     }
 
-    public Mapped extractPersonInfo(ModelIdInput modelIdInput) {
-        String modelId = modelIdInput.getModelId();
-        if (modelId == null) {
-            return Mapped.ERROR("没有选择模型!");
-        }
-        //插入文件
-        LogicalFile logicalFile = createNewLogicalFile(modelId, FNAME_PERSONINFO);
-        String fileId = logicalFile.getId();
-        //创建并存储表头
-        Fields fields = fieldsService.createHeaderAndSave(PERSONINFO_HEADERNAMES, fileId);
-        //根据模型id查询所有的文件
-        String key1 = "fieldsReverse." + FIELDNAME_BANKCARD;
-        Query queryExist = new Query(Criteria.where(key1).exists(true).and(""));
-        List<Fields> fieldsList = mongoTemplate.find(queryExist, Fields.class);
-        for (Fields fields1 : fieldsList) {
-            //再次过滤出是原始非修改过的文件的
-            Query queryIsGenerated = new Query(Criteria.where("id").is(fields1.getFileId()).and("generated").is(Boolean.FALSE));
-            LogicalFile checked = mongoTemplate.findOne(queryIsGenerated, LogicalFile.class);
-            if (checked != null && checked.getId() != null && checked.getId().equals(fields1.getFileId())) {
-                alipayService.extractPersonInfoByFileId(fields1, fields);
+    public void extractPersonInfo(String modelId) {
+        //创建并存储文件和表头
+        Fields output = AlipayHeaderOps.createFileAndFields(AlipayType.ALIPAY_GENERATED_REGISTER, modelId);
+        List<Fields> fieldsList = checkFields(modelId, AlipayType.ALIPAY_ORIGIN_REGISTER);
+        //可能有多个文件,对多个文件并行解析
+        for (Fields origin : fieldsList) {
+            alipayService.extractRegister(origin, output);
+        }
+    }
+
+    private List<Fields> checkFields(String modelId, int requireType) {
+        Query query = new Query(Criteria.where("modelId").is(modelId).and("generated").is(Boolean.FALSE));
+        List<LogicalFile> fileList = mongoTemplate.find(query, LogicalFile.class);
+        List<String> fileIdList = new ArrayList<>();
+        for (LogicalFile file : fileList) {
+            String fileId = file.getId();
+            fileIdList.add(fileId);
+        }
+        Query query1 = new Query(Criteria.where("fileId").in(fileIdList));
+        List<Fields> fieldsList = mongoTemplate.find(query1, Fields.class);
+        List<String> requireHeaders = AlipayHeaders.getHeaders(requireType);
+        List<Fields> checked = new ArrayList<>();
+        for (Fields fields : fieldsList) {
+            List<String> headers = new ArrayList<>(fields.getFieldsReverse().keySet());
+            //字段完全一样就说明是需要查找的表
+            if (headers.containsAll(requireHeaders) && requireHeaders.containsAll(headers)) {
+                checked.add(fields);
             }
         }
-        Mapped mapped = Mapped.OK();
-        mapped.put("result", logicalFile);
-        return mapped;
+        return checked;
+    }
+
+    private MongoCursor<Document> getCursor(String inputFileId) {
+        Query query = new Query(Criteria.where("fileId").is(inputFileId));
+        //查询游标
+        return mongoTemplate.getCollection("data")
+                .find(query.getQueryObject())
+                .batchSize(1000)
+                .noCursorTimeout(true)
+                .cursor();
     }
 
-    //遍历文件里面的所有内容
+    //注册信息
     @Async
-    public void extractPersonInfoByFileId(Fields input, Fields output) {
-        LinkedHashMap<String, String> inputReversed = input.getFieldsReverse();
+    public void extractRegister(Fields origin, Fields output) {
+        LinkedHashMap<String, String> inputReversed = origin.getFieldsReverse();
         LinkedHashMap<String, String> outputReversed = output.getFieldsReverse();
-        String inputFileId = input.getFileId();
+        String inputFileId = origin.getFileId();
         String outputFileId = output.getFileId();
-        String in1 = inputReversed.get("绑定银行卡");
-        String out1 = inputReversed.get("绑定银行卡");
-        String out2 = outputReversed.get("提取银行卡类型");
-        String out3 = outputReversed.get("提取银行卡开户行");
-        String out4 = outputReversed.get("提取银行卡号");
         Query query = new Query(Criteria.where("fileId").is(inputFileId));
         query.fields().include();
+        //查询游标
         MongoCursor<Document> cursor = mongoTemplate.getCollection("data")
                 .find(query.getQueryObject())
                 .batchSize(1000)
                 .noCursorTimeout(true)
                 .cursor();
-        Document doc;
+        Document originDoc;
         List<DataMap> needToSave = new ArrayList<>(1000);
         while (cursor.hasNext()) {
             //写法1（建议）
-            doc = cursor.next();
-            DataMap newDoc = new DataMap();
-            newDoc.put("fileId", outputFileId);
-            //其他字段
-            for (String headerName : PERSONINFO_HEADERNAMES) {
-                String inputAlias = inputReversed.get(headerName);
-                if (StringUtils.isNotBlank(inputAlias)) {
-                    String outputAlias = outputReversed.get(headerName);
-                    if (StringUtils.isNotBlank(outputAlias)) {
-                        newDoc.put(outputAlias, doc.get(inputAlias));
+            originDoc = cursor.next();
+            DataMap newDoc = coryOriginToOutput(originDoc, outputFileId, origin, output);
+            //绑定银行卡字段
+            String aO1 = inputReversed.get("绑定银行卡");
+            String vO1 = originDoc.getString(aO1);
+            String[] sp1 = vO1.split(";");
+            for (String s1 : sp1) {
+                if (StringUtils.isNotBlank(s1)) {
+                    String[] sp2 = s1.split(":");
+                    if (sp2.length == 3) {
+                        //拷贝一行新的
+                        DataMap newLine = new DataMap(newDoc);
+                        newLine.put(outputReversed.getOrDefault(AlipayHeaders.REGISTER_1, "-1"), sp2[0]);
+                        newLine.put(outputReversed.getOrDefault(AlipayHeaders.REGISTER_2, "-1"), sp2[1]);
+                        newLine.put(outputReversed.getOrDefault(AlipayHeaders.REGISTER_3, "-1"), sp2[2]);
+                        needToSave.add(newLine);
                     }
                 }
             }
+            if (needToSave.size() >= 1000) {
+                saveLines(new ArrayList<>(needToSave));
+                needToSave.clear();
+            }
+        }
+        if (CollectionUtils.isNotEmpty(needToSave)) {
+            saveLines(needToSave);
+        }
+    }
+
+    @Async
+    public void extractLogin(MongoCursor<Document> cursor, Fields origin, Fields output) {
+        LinkedHashMap<String, String> inputReversed = origin.getFieldsReverse();
+        LinkedHashMap<String, String> outputReversed = output.getFieldsReverse();
+        Document originDoc;
+        List<DataMap> needToSave = new ArrayList<>(1000);
+        while (cursor.hasNext()) {
+            //写法1（建议）
+            originDoc = cursor.next();
+            DataMap newDoc = coryOriginToOutput(originDoc, output.getFileId(), origin, output);
             //绑定银行卡字段
-            String value1 = doc.getString(in1);
-            String[] splited1 = value1.split(";");
-            for (String s1 : splited1) {
-                if (StringUtils.isNotBlank(s1)) {
-                    String[] splited2 = s1.split(":");
-                    if (splited2.length == 3) {
-                        newDoc.put(out2, splited2[0]);
-                        newDoc.put(out3, splited2[1]);
-                        newDoc.put(out4, splited2[2]);
-                        needToSave.add(newDoc);
+            String aO1 = inputReversed.get("操作发生时间");
+            String vO1 = originDoc.getString(aO1);
+            newDoc.put(outputReversed.getOrDefault(AlipayHeaders.LOGIN_1, "-1"), DataUtils.getDateFront(vO1));
+            newDoc.put(outputReversed.getOrDefault(AlipayHeaders.LOGIN_2, "-1"), DataUtils.getDateEnd(vO1));
+            needToSave.add(newDoc);
+            if (needToSave.size() >= 1000) {
+                saveLines(new ArrayList<>(needToSave));
+                needToSave.clear();
+            }
+        }
+        if (CollectionUtils.isNotEmpty(needToSave)) {
+            saveLines(needToSave);
+        }
+    }
+
+    @Async
+    public void extractTrade(MongoCursor<Document> cursor, Fields origin, Fields output) {
+        LinkedHashMap<String, String> inputReversed = origin.getFieldsReverse();
+        LinkedHashMap<String, String> outputReversed = output.getFieldsReverse();
+        Document originDoc;
+        List<DataMap> needToSave = new ArrayList<>(1000);
+        while (cursor.hasNext()) {
+            //写法1（建议）
+            originDoc = cursor.next();
+            DataMap newDoc = coryOriginToOutput(originDoc, output.getFileId(), origin, output);
+            //绑定银行卡字段
+            String i1 = inputReversed.get("买家信息");
+            String v1 = originDoc.getString(i1);
+            String i2 = inputReversed.get("卖家信息");
+            String v2 = originDoc.getString(i2);
+            if (StringUtils.isNotBlank(v1)) {
+                int firstIndex = 0;
+                int secondIndex = 0;
+                String alias1 = outputReversed.getOrDefault(AlipayHeaders.TRADE_1, "-1");
+                String alias2 = outputReversed.getOrDefault(AlipayHeaders.TRADE_2, "-1");
+                String alias3 = outputReversed.getOrDefault(AlipayHeaders.TRADE_3, "-1");
+                StringBuilder o1 = new StringBuilder();
+                StringBuilder o2 = new StringBuilder();
+                StringBuilder o3 = new StringBuilder();
+                for (int i = 0; i < v1.length(); i++) {
+                    char current = v1.charAt(i);
+                    if (current != '(' && current != ')') {
+                        if (firstIndex == 0 && secondIndex == 0) {
+                            o1.append(current);
+                        } else if (firstIndex == 1 && secondIndex == 0) {
+                            o2.append(current);
+                        } else if (firstIndex == 1 && secondIndex == 1) {
+                            o3.append(current);
+                        }
+                    } else {
+                        if (firstIndex == 0 && current == '(') {
+                            firstIndex = 1;
+                        }else if(firstIndex == 1 && secondIndex == 0 && current == '('){
+                            secondIndex = 1;
+                        }
                     }
                 }
+                newDoc.put(alias1, o1.toString());
+                newDoc.put(alias2, o2.toString());
+                newDoc.put(alias3, o3.toString());
             }
+            if (StringUtils.isNotBlank(v2)) {
+                int firstIndex = 0;
+                int secondIndex = 0;
+                String alias1 = outputReversed.getOrDefault(AlipayHeaders.TRADE_4, "-1");
+                String alias2 = outputReversed.getOrDefault(AlipayHeaders.TRADE_5, "-1");
+                String alias3 = outputReversed.getOrDefault(AlipayHeaders.TRADE_6, "-1");
+                StringBuilder o1 = new StringBuilder();
+                StringBuilder o2 = new StringBuilder();
+                StringBuilder o3 = new StringBuilder();
+                for (int i = 0; i < v2.length(); i++) {
+                    char current = v2.charAt(i);
+                    if (current != '(' && current != ')') {
+                        if (firstIndex == 0 && secondIndex == 0) {
+                            o1.append(current);
+                        } else if (firstIndex == 1 && secondIndex == 0) {
+                            o2.append(current);
+                        } else if (firstIndex == 1 && secondIndex == 1) {
+                            o3.append(current);
+                        }
+                    } else {
+                        if (firstIndex == 0 && current == '(') {
+                            firstIndex = 1;
+                        }else if(firstIndex == 1 && secondIndex == 0 && current == '('){
+
+                            secondIndex = 1;
+                        }
+                    }
+                }
+                newDoc.put(alias1, o1.toString());
+                newDoc.put(alias2, o2.toString());
+                newDoc.put(alias3, o3.toString());
+            }
+            needToSave.add(newDoc);
             if (needToSave.size() >= 1000) {
                 saveLines(new ArrayList<>(needToSave));
                 needToSave.clear();
             }
         }
         if (CollectionUtils.isNotEmpty(needToSave)) {
-//            saveLines(new ArrayList<>(needToSave));
             saveLines(needToSave);
         }
     }
 
-    private void readFieldBankCard() {
+    @Async
+    public void extractAccount(MongoCursor<Document> cursor, Fields origin, Fields output) {
+        LinkedHashMap<String, String> inputReversed = origin.getFieldsReverse();
+        LinkedHashMap<String, String> outputReversed = output.getFieldsReverse();
+        Document originDoc;
+        List<DataMap> needToSave = new ArrayList<>(1000);
+        while (cursor.hasNext()) {
+            //写法1（建议）
+            originDoc = cursor.next();
+            DataMap newDoc = coryOriginToOutput(originDoc, output.getFileId(), origin, output);
+            needToSave.add(newDoc);
+            if (needToSave.size() >= 1000) {
+                saveLines(new ArrayList<>(needToSave));
+                needToSave.clear();
+            }
+        }
+        if (CollectionUtils.isNotEmpty(needToSave)) {
+            saveLines(needToSave);
+        }
+    }
 
+    @Async
+    public void extractTransfer(MongoCursor<Document> cursor, Fields origin, Fields output) {
+        LinkedHashMap<String, String> inputReversed = origin.getFieldsReverse();
+        LinkedHashMap<String, String> outputReversed = output.getFieldsReverse();
+        Document originDoc;
+        List<DataMap> needToSave = new ArrayList<>(1000);
+        while (cursor.hasNext()) {
+            //写法1（建议）
+            originDoc = cursor.next();
+            DataMap newDoc = coryOriginToOutput(originDoc, output.getFileId(), origin, output);
+            //绑定银行卡字段
+            needToSave.add(newDoc);
+            if (needToSave.size() >= 1000) {
+                saveLines(new ArrayList<>(needToSave));
+                needToSave.clear();
+            }
+        }
+        if (CollectionUtils.isNotEmpty(needToSave)) {
+            saveLines(needToSave);
+        }
     }
 
-    /**
-     * 创建新的逻辑表,只包含表头
-     */
-    public LogicalFile createNewLogicalFile(String modelId, String filename) {
-        LogicalFile logicalFile = new LogicalFile();
-        logicalFile.setFilename(filename);
-        logicalFile.setStatus(FileStatus.STATUS_LOADING);
-        String current = DataUtils.currentDate();
-        logicalFile.setUploadDate(current);
-        logicalFile.setModelId(modelId);
-        logicalFile.setGenerated(Boolean.TRUE);
-        return mongoTemplate.insert(logicalFile);
+    //复制原始数据到新的一行
+    private DataMap coryOriginToOutput(Document originData, String newFileId, Fields originFields, Fields outputFields) {
+        DataMap newData = new DataMap();
+        newData.put("fileId", newFileId);
+        //遍历新的表的每一个字段
+        LinkedHashMap<String, String> originHeaderReversed = originFields.getFieldsReverse();
+        LinkedHashMap<String, String> newHeaderReversed = outputFields.getFieldsReverse();
+        for (Map.Entry<String, String> entry : newHeaderReversed.entrySet()) {
+            String headerName = entry.getKey();
+            String alias = entry.getValue();
+            String originAlias = originHeaderReversed.getOrDefault(headerName, "");
+            newData.put(alias, originData.getOrDefault(originAlias, ""));
+        }
+        return newData;
     }
 
     @Async

src/main/java/ieven/server/webapp/domain/alipay/AlipayType.java

@@ -0,0 +1,25 @@
+package ieven.server.webapp.domain.alipay;
+
+//表类型
+public class AlipayType {
+    //支付宝注册信息原表
+    public final static int ALIPAY_ORIGIN_REGISTER = 1;
+    //支付宝生成的用户信息表
+    public final static int ALIPAY_GENERATED_REGISTER = 2;
+    //支付宝登录日志原表
+    public final static int ALIPAY_ORIGIN_LOGIN = 3;
+    //支付宝登录日志提取表
+    public final static int ALIPAY_GENERATED_LOGIN = 4;
+    //支付宝交易记录原表
+    public final static int ALIPAY_ORIGIN_TRADE = 5;
+    //支付宝交易记录提取表
+    public final static int ALIPAY_GENERATED_TRADE = 6;
+    //支付宝登录账户明细原表
+    public final static int ALIPAY_ORIGIN_ACCOUNT = 7;
+    //支付宝登录账户明细提取表
+    public final static int ALIPAY_GENERATED_ACCOUNT = 8;
+    //支付宝登录转账明细原表
+    public final static int ALIPAY_ORIGIN_TRANSFER = 9;
+    //支付宝登录转账明细提取表
+    public final static int ALIPAY_GENERATED_TRANSFER = 10;
+}

src/main/java/ieven/server/webapp/domain/alipay/OperateInput.java

@@ -1,12 +0,0 @@
-package ieven.server.webapp.domain.alipay;
-
-import lombok.Data;
-
-import java.util.List;
-
-@Data
-public class OperateInput {
-    private String inputType;
-    private List<String> selectedFileIds;
-    private List<String> selectedFieldsForAlias;
-}

src/main/java/ieven/server/webapp/domain/common/HeaderModifier.java

@@ -1,4 +0,0 @@
-package ieven.server.webapp.domain.common;
-
-public class HeaderModifier {
-}

src/main/java/ieven/server/webapp/domain/common/TableHeaders.java

@@ -1,11 +0,0 @@
-package ieven.server.webapp.domain.common;
-
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * 固定的表头
- */
-public class TableHeaders {
-    public final static List<String> FIELD_LIST_REGISTER = Arrays.asList("用户ID", "登录邮箱", "登录手机", "账户名称", "证件类型", "证件号", "可用余额", "绑定手机", "绑定银行卡", "对应的协查数据");
-}

src/main/java/ieven/server/webapp/domain/common/TableType.java

@@ -1,7 +0,0 @@
-package ieven.server.webapp.domain.common;
-
-//表类型
-public class TableType {
-    //支付宝注册信息原表
-    public final static int ALIPAY_ORIGIN_REGISTER = 1;
-}

src/main/java/ieven/server/webapp/domain/data/DataMap.java

@@ -7,6 +7,14 @@ import java.util.HashMap;
 
 @Document(collection = "data")
 public class DataMap extends HashMap<String, Object> {
+    public DataMap(DataMap dataMap) {
+        super(dataMap);
+    }
+
+    public DataMap() {
+        super();
+    }
+
     public void formId() {
         ObjectId id = (ObjectId) this.getOrDefault("_id", null);
         if (id != null) {

src/main/java/ieven/server/webapp/domain/data/DataService.java

@@ -5,9 +5,11 @@ import ieven.server.webapp.domain.IdInput;
 import ieven.server.webapp.infrastructure.wrapper.Mapped;
 import ieven.server.webapp.util.Properties;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.bson.types.ObjectId;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.data.domain.Sort;
 import org.springframework.data.mongodb.core.MongoTemplate;
 import org.springframework.data.mongodb.core.query.Criteria;
 import org.springframework.data.mongodb.core.query.Query;
@@ -18,6 +20,7 @@ import org.springframework.stereotype.Service;
 import java.util.*;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
+import java.util.regex.Pattern;
 
 @Slf4j
 @Service
@@ -62,6 +65,87 @@ public class DataService {
         }
     }
 
+    public Mapped viewDataByQueryParam(QueryParam param) {
+        String fileId = param.getFileId();
+        int page = param.getPage();
+        int pageSize = param.getPageSize();
+        int skip = (page - 1) * pageSize;
+        String mode = param.getMode();
+        Criteria criteria = Criteria.where("fileId").is(fileId);
+        List<QueryParam.Param1> param1List = param.getValues();
+        List<Criteria> criteriaList = new ArrayList<>();
+        if (CollectionUtils.isNotEmpty(param1List)) {
+            for (QueryParam.Param1 param1 : param1List) {
+                String fieldAlias = param1.getFieldAlias();
+                String value = param1.getValue();
+                if (StringUtils.isBlank(fieldAlias)) {
+                    continue;
+                }
+                Criteria criteria1 = Criteria.where(fieldAlias);
+                String mode1 = param1.getMode();
+                if ("is".equals(mode1)) {
+                    criteria1.is(value);
+                } else {
+                    String regex = "^.*" + value + ".*$";
+                    Pattern pattern = Pattern.compile(regex, Pattern.CASE_INSENSITIVE);
+                    criteria1.regex(pattern);
+                }
+                if (CollectionUtils.isNotEmpty(criteriaList)) {
+                    Criteria last = criteriaList.get(criteriaList.size() - 1);
+                    if ("and".equals(mode)) {
+                        last.andOperator(criteria1);
+                    } else if ("or".equals(mode)) {
+                        last.orOperator(criteria1);
+                    }
+                }
+                criteriaList.add(criteria1);
+            }
+
+        }
+        if (CollectionUtils.isNotEmpty(criteriaList)) {
+            criteria.andOperator(criteriaList.get(0));
+        }
+        Future<Long> totalFuture = countTotalByQuery(new Query(criteria));
+        Future<LinkedHashMap<String, String>> headerFuture = viewHeaders(fileId);
+        Query dataQuery = new Query(criteria);
+        if (CollectionUtils.isNotEmpty(param.getOrders())) {
+            for (QueryParam.Param2 param2 : param.getOrders()) {
+                String alias = param2.getFieldAlias();
+                String orderBy = param2.getMode();
+                if ("asc".equals(orderBy)) {
+                    dataQuery.with(Sort.by(Sort.Order.asc(alias)));
+                } else if ("desc".equals(orderBy)) {
+                    dataQuery.with(Sort.by(Sort.Order.desc(alias)));
+                }
+            }
+        }
+        //分页
+        dataQuery.skip(skip).limit(pageSize);
+        List<DataMap> resultList = mongoTemplate.find(
+                dataQuery,
+                DataMap.class);
+        for (DataMap dataMap : resultList) {
+            dataMap.formId();
+        }
+        Long total = null;
+        LinkedHashMap<String, String> headers = null;
+        try {
+            total = totalFuture.get();
+            headers = headerFuture.get();
+        } catch (InterruptedException | ExecutionException e) {
+            e.printStackTrace();
+            return Mapped.ERROR("系统查询出错");
+        }
+        Mapped mapped = Mapped.OK();
+        mapped.put("result", resultList);
+        mapped.put("header", headers);
+        mapped.put("code", 200);
+        mapped.put("page", page);
+        mapped.put("total", total);
+        mapped.put("pageSize", pageSize);
+        return mapped;
+    }
+
     public Mapped viewHeaders(FileIdInput fileIdInput) {
         String fileId = fileIdInput.getFileId();
         Fields fields = mongoTemplate.findOne(new Query(Criteria.where("fileId").is(fileId)), Fields.class);
@@ -76,6 +160,12 @@ public class DataService {
         return new AsyncResult<>(total);
     }
 
+    @Async
+    Future<Long> countTotalByQuery(Query query) {
+        Long total = mongoTemplate.count(query, DataMap.class);
+        return new AsyncResult<>(total);
+    }
+
     @Async
     Future<LinkedHashMap<String, String>> viewHeaders(String fileId) {
         Query query = new Query(Criteria.where("fileId").is(fileId));

src/main/java/ieven/server/webapp/domain/data/Fields.java

@@ -23,4 +23,5 @@ public class Fields {
     private String fileId;
     private LinkedHashMap<String, String> fields;
     private LinkedHashMap<String, String> fieldsReverse;
+    private Integer tableType;
 }

src/main/java/ieven/server/webapp/domain/data/FieldsService.java

@@ -1,31 +0,0 @@
-package ieven.server.webapp.domain.data;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.data.mongodb.core.MongoTemplate;
-import org.springframework.stereotype.Service;
-
-import java.util.LinkedHashMap;
-import java.util.List;
-
-@Service
-public class FieldsService {
-    @Autowired
-    private MongoTemplate mongoTemplate;
-
-    public Fields createHeaderAndSave(List<String> headerNames, String fileId) {
-        Fields fields = new Fields();
-        fields.setFileId(fileId);
-        //插入表头
-        LinkedHashMap<String, String> header = new LinkedHashMap<>();
-        LinkedHashMap<String, String> headerReversed = new LinkedHashMap<>();
-        for (int i = 0; i < headerNames.size(); i++) {
-            String field = headerNames.get(i);
-            String alias = "f" + i;
-            header.put(alias, field);
-            headerReversed.put(field, alias);
-        }
-        fields.setFields(header);
-        fields.setFieldsReverse(headerReversed);
-        return mongoTemplate.insert(fields);
-    }
-}

src/main/java/ieven/server/webapp/domain/data/QueryParam.java

@@ -0,0 +1,33 @@
+package ieven.server.webapp.domain.data;
+
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+public class QueryParam {
+    private String fileId;
+    private List<Param1> values;
+    private List<Param2> orders;
+    private Integer page;
+    private Integer pageSize;
+    //and or
+    private String mode;
+
+    @Data
+    public static class Param1 {
+        private String field;
+        private String fieldAlias;
+        private String value;
+        //is like
+        private String mode;
+    }
+
+    @Data
+    public static class Param2 {
+        private String field;
+        private String fieldAlias;
+        //asc desc
+        private String mode;
+    }
+}

src/main/java/ieven/server/webapp/domain/file/FileService.java

@@ -20,6 +20,7 @@ import org.bson.types.ObjectId;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.context.annotation.Lazy;
+import org.springframework.data.domain.Sort;
 import org.springframework.data.mongodb.core.MongoTemplate;
 import org.springframework.data.mongodb.core.query.Criteria;
 import org.springframework.data.mongodb.core.query.Query;
@@ -142,7 +143,7 @@ public class FileService {
 
     public Mapped listUploaded(FileListInput fileListInput) {
         String modelId = fileListInput.getModelId();
-        if(modelId==null){
+        if (modelId == null) {
             return Mapped.ERROR("没有选择任何分组");
         }
         Future<Long> longFuture = fileService.countTotalByModelId(modelId);
@@ -151,6 +152,9 @@ public class FileService {
         int pageSize = fileListInput.getPageSize();
         int skip = (page - 1) * pageSize;
         Query query = new Query(Criteria.where("modelId").is(modelId));
+        query.with(Sort.by(
+                Sort.Order.desc("uploadDate")
+        ));
         List<LogicalFile> resultList = mongoTemplate.find(
                 query.skip(skip).limit(pageSize),
                 LogicalFile.class);
@@ -162,21 +166,23 @@ public class FileService {
         } catch (InterruptedException | ExecutionException e) {
             return Mapped.ERROR("查询出错");
         }
+        int i = 0;
         for (LogicalFile file : resultList) {
             file.setModelName(modelName);
+            file.setCount(++i);
         }
         Mapped res = Mapped.OK();
         res.put("code", 200);
         res.put("result", resultList);
         res.put("total", total);
-        res.put("page", page);
+        res.put("currentPage", page);
         res.put("pageSize", pageSize);
         return res;
     }
 
     public Mapped deleteById(IdInput idInput) {
         String id = idInput.getId();
-        Query query = new Query(Criteria.where("_id").is(id));
+        Query query = new Query(Criteria.where("id").is(id));
         LogicalFile file = mongoTemplate.findOne(query, LogicalFile.class);
         //删除物理文件
         if (file != null && file.getGridId() != null) {
@@ -211,12 +217,10 @@ public class FileService {
 
     @Async
     Future<String> retrieveModelname(String modelId) {
-        Model model = mongoTemplate.findOne(new Query(Criteria.where("id").is(modelId)), Model.class);
-        String modelName = "";
-        if (model != null) {
-            modelName = model.getModelName();
-        }
-        return new AsyncResult<>(modelName);
+        Query query = new Query(Criteria.where("id").is(modelId));
+        query.fields().include("modelName");
+        Model model = mongoTemplate.findOne(query, Model.class, "model");
+        return new AsyncResult<>(model != null ? model.getModelName() : "");
     }
 
 }

src/main/java/ieven/server/webapp/domain/file/LogicalFile.java

@@ -16,6 +16,7 @@ public class LogicalFile {
     private String modelId;
     @JsonIgnore
     private ObjectId gridId;
+    private Integer count;
     private String status;
     private String filename;
     private Boolean generated;

src/main/java/ieven/server/webapp/util/DataUtils.java

@@ -1,10 +1,40 @@
 package ieven.server.webapp.util;
 
+import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Date;
 
 public class DataUtils {
-    public static String currentDate(){
+    public static String currentDate() {
         return new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date());
     }
+
+    public static Date transferData(String str) {
+        SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+        Date date = null;
+        try {
+            date = format.parse(str);
+        } catch (ParseException e) {
+            return null;
+        }
+        return date;
+    }
+
+    public static String getDateFront(String str) {
+        Date date = transferData(str);
+        if (date != null) {
+            SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
+            return format.format(date);
+        }
+        return "";
+    }
+
+    public static String getDateEnd(String str) {
+        Date date = transferData(str);
+        if (date != null) {
+            SimpleDateFormat format = new SimpleDateFormat("HH:mm:ss");
+            return format.format(date);
+        }
+        return "";
+    }
 }