提交修复BUG

src/main/java/ieven/server/webapp/api/AlipayController.java

@@ -5,6 +5,7 @@ import ieven.server.webapp.domain.alipay.AlipayService;
 import ieven.server.webapp.infrastructure.wrapper.Mapped;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
 
@@ -17,30 +18,11 @@ public class AlipayController {
     @Autowired
     private AlipayService alipayService;
 
-    @PostMapping("/operations")
-    public Mapped listOperations() {
-        Map<String, Object> operations = new LinkedHashMap<>();
-        alipayService.testCursor();
-        return null;
-    }
-
-    @PostMapping("/operate")
-    public Mapped operate() {
-        Map<String, Object> operations = new LinkedHashMap<>();
-        return null;
-    }
-
-    @PostMapping("/operate_chain")
-    public Mapped operateChain() {
-        Map<String, Object> operations = new LinkedHashMap<>();
-        return null;
-    }
-
     /**
      * 提取人员信息.
      */
     @PostMapping("/extractPersonInfo")
-    public Mapped extractPersonInfo(ModelIdInput modelIdInput){
+    public Mapped extractPersonInfo(@RequestBody ModelIdInput modelIdInput) {
         return alipayService.extractPersonInfo(modelIdInput);
     }
 }

src/main/java/ieven/server/webapp/domain/alipay/AlipayService.java

@@ -2,6 +2,7 @@ package ieven.server.webapp.domain.alipay;
 
 import com.mongodb.client.MongoCursor;
 import ieven.server.webapp.domain.ModelIdInput;
+import ieven.server.webapp.domain.data.DataMap;
 import ieven.server.webapp.domain.data.Fields;
 import ieven.server.webapp.domain.data.FieldsService;
 import ieven.server.webapp.domain.file.FileStatus;
@@ -10,19 +11,18 @@ import ieven.server.webapp.infrastructure.wrapper.Mapped;
 import ieven.server.webapp.util.DataUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.ObjectUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.bson.Document;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Lazy;
 import org.springframework.data.mongodb.core.MongoTemplate;
 import org.springframework.data.mongodb.core.query.Criteria;
 import org.springframework.data.mongodb.core.query.Query;
 import org.springframework.scheduling.annotation.Async;
 import org.springframework.stereotype.Service;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.LinkedHashMap;
-import java.util.List;
+import java.util.*;
 
 @Slf4j
 @Service
@@ -31,6 +31,9 @@ public class AlipayService {
     private MongoTemplate mongoTemplate;
     @Autowired
     private FieldsService fieldsService;
+    @Autowired
+    @Lazy
+    private AlipayService alipayService;
     private final static String FNAME_PERSONINFO = "提取的人员信息表";
     private final static String FIELDNAME_BANKCARD = "绑定银行卡";
     private final static String EXTRACT_BANKCARDTYPE = "提取银行卡类型";
@@ -63,10 +66,15 @@ public class AlipayService {
         Fields fields = fieldsService.createHeaderAndSave(PERSONINFO_HEADERNAMES, fileId);
         //根据模型id查询所有的文件
         String key1 = "fieldsReverse." + FIELDNAME_BANKCARD;
-        Query queryExist = new Query(Criteria.where(key1).exists(true));
+        Query queryExist = new Query(Criteria.where(key1).exists(true).and(""));
         List<Fields> fieldsList = mongoTemplate.find(queryExist, Fields.class);
         for (Fields fields1 : fieldsList) {
-            extractPersonInfoByFileId(fields1);
+            //再次过滤出是原始非修改过的文件的
+            Query queryIsGenerated = new Query(Criteria.where("id").is(fields1.getFileId()).and("generated").is(Boolean.FALSE));
+            LogicalFile checked = mongoTemplate.findOne(queryIsGenerated, LogicalFile.class);
+            if (checked != null && checked.getId() != null && checked.getId().equals(fields1.getFileId())) {
+                alipayService.extractPersonInfoByFileId(fields1, fields);
+            }
         }
         Mapped mapped = Mapped.OK();
         mapped.put("result", logicalFile);
@@ -75,11 +83,17 @@ public class AlipayService {
 
     //遍历文件里面的所有内容
     @Async
-    public void extractPersonInfoByFileId(Fields fields) {
-        LinkedHashMap<String, String> headerReversed = fields.getFieldsReverse();
-        String fField1 = headerReversed.get(FIELDNAME_BANKCARD);
-        String fileId = fields.getFileId();
-        Query query = new Query(Criteria.where("fileId").is(fileId));
+    public void extractPersonInfoByFileId(Fields input, Fields output) {
+        LinkedHashMap<String, String> inputReversed = input.getFieldsReverse();
+        LinkedHashMap<String, String> outputReversed = output.getFieldsReverse();
+        String inputFileId = input.getFileId();
+        String outputFileId = output.getFileId();
+        String in1 = inputReversed.get("绑定银行卡");
+        String out1 = inputReversed.get("绑定银行卡");
+        String out2 = outputReversed.get("提取银行卡类型");
+        String out3 = outputReversed.get("提取银行卡开户行");
+        String out4 = outputReversed.get("提取银行卡号");
+        Query query = new Query(Criteria.where("fileId").is(inputFileId));
         query.fields().include();
         MongoCursor<Document> cursor = mongoTemplate.getCollection("data")
                 .find(query.getQueryObject())
@@ -87,22 +101,33 @@ public class AlipayService {
                 .noCursorTimeout(true)
                 .cursor();
         Document doc;
-        List<Document> needToSave = new ArrayList<>(1000);
+        List<DataMap> needToSave = new ArrayList<>(1000);
         while (cursor.hasNext()) {
             //写法1（建议）
             doc = cursor.next();
-            //绑定银行卡
-            String value1 = doc.getString(fField1);
+            DataMap newDoc = new DataMap();
+            newDoc.put("fileId", outputFileId);
+            //其他字段
+            for (String headerName : PERSONINFO_HEADERNAMES) {
+                String inputAlias = inputReversed.get(headerName);
+                if (StringUtils.isNotBlank(inputAlias)) {
+                    String outputAlias = outputReversed.get(headerName);
+                    if (StringUtils.isNotBlank(outputAlias)) {
+                        newDoc.put(outputAlias, doc.get(inputAlias));
+                    }
+                }
+            }
+            //绑定银行卡字段
+            String value1 = doc.getString(in1);
             String[] splited1 = value1.split(";");
             for (String s1 : splited1) {
                 if (StringUtils.isNotBlank(s1)) {
                     String[] splited2 = s1.split(":");
                     if (splited2.length == 3) {
-                        doc.put(EXTRACT_BANKCARDTYPE, splited2[0]);
-                        doc.put(EXTRACT_BANKCARDCODE, splited2[1]);
-                        doc.put(EXTRACT_BANKCARD, splited2[2]);
-                        doc.remove("_id");
-                        needToSave.add(doc);
+                        newDoc.put(out2, splited2[0]);
+                        newDoc.put(out3, splited2[1]);
+                        newDoc.put(out4, splited2[2]);
+                        needToSave.add(newDoc);
                     }
                 }
             }
@@ -112,6 +137,7 @@ public class AlipayService {
             }
         }
         if (CollectionUtils.isNotEmpty(needToSave)) {
+//            saveLines(new ArrayList<>(needToSave));
             saveLines(needToSave);
         }
     }
@@ -126,11 +152,13 @@ public class AlipayService {
         String current = DataUtils.currentDate();
         logicalFile.setUploadDate(current);
         logicalFile.setModelId(modelId);
+        logicalFile.setGenerated(Boolean.TRUE);
         return mongoTemplate.insert(logicalFile);
     }
 
     @Async
-    public void saveLines(List<Document> documentList) {
-        mongoTemplate.insert(documentList, "data1");
+    public void saveLines(List<DataMap> documentList) {
+        mongoTemplate.insert(documentList,DataMap.class);
     }
+
 }

src/main/java/ieven/server/webapp/domain/file/FileService.java

@@ -5,10 +5,7 @@ import com.mongodb.client.result.DeleteResult;
 import ieven.server.webapp.domain.IdInput;
 import ieven.server.webapp.domain.data.DataMap;
 import ieven.server.webapp.infrastructure.wrapper.Mapped;
-import ieven.server.webapp.model.UploadedFile;
-import ieven.server.webapp.model.display.UploadedFileDisplay;
 import ieven.server.webapp.service.impl.MongoExcelServiceImpl;
-import ieven.server.webapp.util.Properties;
 import ieven.server.webapp.util.excel.ExcelXlsReader;
 import ieven.server.webapp.util.excel.ExcelXlsxReader;
 import ieven.server.webapp.util.excel.PublicStatic;
@@ -21,6 +18,7 @@ import org.apache.poi.openxml4j.exceptions.OpenXML4JException;
 import org.bson.types.ObjectId;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.context.annotation.Lazy;
 import org.springframework.data.mongodb.core.MongoTemplate;
 import org.springframework.data.mongodb.core.query.Criteria;
 import org.springframework.data.mongodb.core.query.Query;
@@ -30,6 +28,7 @@ import org.springframework.data.mongodb.gridfs.GridFsResource;
 import org.springframework.data.mongodb.gridfs.GridFsTemplate;
 import org.springframework.scheduling.annotation.Async;
 import org.springframework.scheduling.annotation.AsyncResult;
+import org.springframework.scheduling.annotation.EnableAsync;
 import org.springframework.stereotype.Service;
 import org.springframework.web.multipart.MultipartFile;
 import org.xml.sax.SAXException;
@@ -39,9 +38,7 @@ import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.text.SimpleDateFormat;
 import java.util.Date;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 import java.util.concurrent.ThreadPoolExecutor;
@@ -49,6 +46,7 @@ import java.util.stream.Collectors;
 
 @Slf4j
 @Service
+@EnableAsync
 public class FileService {
     @Autowired
     private GridFsTemplate gridFsTemplate;
@@ -59,6 +57,9 @@ public class FileService {
     private ThreadPoolExecutor threadPoolExecutor;
     @Autowired
     private GridFsOperations fsOperations;
+    @Autowired
+    @Lazy
+    private FileService fileService;
 
     public Mapped storeUploaded(MultipartFile file, String modelId) {
         String filename = file.getOriginalFilename();
@@ -77,11 +78,12 @@ public class FileService {
         logicalFile.setUploadDate(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()));
         logicalFile.setModelId(modelId);
         logicalFile.setGridId(objectId);
+        logicalFile.setGenerated(Boolean.FALSE);
         logicalFile.setStatus(FileStatus.STATUS_LOADING);
         //使用主键自动生成id
         logicalFile = mongoTemplate.insert(logicalFile);
         //异步读取数据并存储到mongo
-        readFile(logicalFile);
+        fileService.readFile(logicalFile);
 //        fileService.readFileWhileStoreDataAsync(objectId);
         return Mapped.OK();
     }
@@ -142,7 +144,7 @@ public class FileService {
 
     public Mapped listUploaded(FileListInput fileListInput) {
         String modelId = fileListInput.getModelId();
-        Future<Long> longFuture = countTotalByModelId(modelId);
+        Future<Long> longFuture = fileService.countTotalByModelId(modelId);
         int page = fileListInput.getPage();
         int pageSize = fileListInput.getPageSize();
         int skip = (page - 1) * pageSize;
@@ -172,10 +174,10 @@ public class FileService {
         //删除物理文件
         if (file != null && file.getGridId() != null) {
             String physicalId = file.getGridId().toHexString();
-            removePhysicalFile(physicalId);
+            fileService.removePhysicalFile(physicalId);
         }
         //删除数据
-        removeDataByFileId(id);
+        fileService.removeDataByFileId(id);
         DeleteResult result = mongoTemplate.remove(query, LogicalFile.class);
         if (result.getDeletedCount() > 0) {
             return Mapped.OK();
@@ -199,4 +201,5 @@ public class FileService {
     void removeDataByFileId(String fileId) {
         mongoTemplate.remove(new Query(Criteria.where("fileId").is(fileId)), DataMap.class);
     }
+
 }

src/main/java/ieven/server/webapp/domain/file/FileServiceAsync.java

@@ -0,0 +1,110 @@
+package ieven.server.webapp.domain.file;
+
+import com.mongodb.client.gridfs.model.GridFSFile;
+import ieven.server.webapp.domain.data.DataMap;
+import ieven.server.webapp.service.impl.MongoExcelServiceImpl;
+import ieven.server.webapp.util.excel.ExcelXlsReader;
+import ieven.server.webapp.util.excel.ExcelXlsxReader;
+import ieven.server.webapp.util.excel.PublicStatic;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVParser;
+import org.apache.commons.csv.CSVRecord;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.poi.openxml4j.exceptions.OpenXML4JException;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.data.mongodb.core.MongoTemplate;
+import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.data.mongodb.core.query.Query;
+import org.springframework.data.mongodb.core.query.Update;
+import org.springframework.data.mongodb.gridfs.GridFsOperations;
+import org.springframework.data.mongodb.gridfs.GridFsResource;
+import org.springframework.scheduling.annotation.Async;
+import org.springframework.scheduling.annotation.AsyncResult;
+import org.springframework.scheduling.annotation.EnableAsync;
+import org.springframework.stereotype.Service;
+import org.xml.sax.SAXException;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.concurrent.Future;
+import java.util.stream.Collectors;
+
+@Service
+public class FileServiceAsync {
+    @Autowired
+    private MongoTemplate mongoTemplate;
+    @Autowired
+    private GridFsOperations fsOperations;
+
+    @Async
+    public void readFile(LogicalFile uploaded) {
+        GridFSFile fsFile = fsOperations.findOne(new Query(Criteria.where("_id").is(uploaded.getGridId())));
+        if (fsFile == null) {
+            //解析失败
+            modifyStatus(uploaded.getId(), FileStatus.STATUS_ERROR);
+            return;
+        }
+        GridFsResource resource = fsOperations.getResource(fsFile);
+        String filename = resource.getFilename();
+        if (StringUtils.isBlank(filename)) {
+            //解析失败
+            modifyStatus(uploaded.getId(), FileStatus.STATUS_ERROR);
+            return;
+        }
+        try {
+            InputStream inputStream = resource.getInputStream();
+            if (filename.endsWith(".xlsx")) {
+                MongoExcelServiceImpl mongoExcelService = new MongoExcelServiceImpl(mongoTemplate, uploaded.getId());
+                ExcelXlsxReader reader = new ExcelXlsxReader(mongoExcelService);
+                reader.processStream(inputStream);
+                mongoExcelService.insertRest();
+            } else if (filename.endsWith(".xls")) {
+                MongoExcelServiceImpl mongoExcelService = new MongoExcelServiceImpl(mongoTemplate, uploaded.getId());
+                ExcelXlsReader reader = new ExcelXlsReader(mongoExcelService);
+                reader.processStream(inputStream);
+                mongoExcelService.insertRest();
+            } else if (filename.endsWith(".csv")) {
+                MongoExcelServiceImpl mongoExcelService = new MongoExcelServiceImpl(mongoTemplate, uploaded.getId());
+                CSVParser csvParser = CSVFormat.EXCEL.parse(new InputStreamReader(inputStream, "gbk"));
+                int curRow = 0;
+                for (CSVRecord record : csvParser) {
+                    mongoExcelService.getRows(0, curRow, record.stream().map(PublicStatic::removeAllSpecial).collect(Collectors.toList()));
+                    curRow++;
+                }
+                mongoExcelService.insertRest();
+            }
+        } catch (IOException | OpenXML4JException | SAXException e) {
+            //解析失败
+            modifyStatus(uploaded.getId(), FileStatus.STATUS_ERROR);
+            e.printStackTrace();
+        }
+        //解析成功
+        modifyStatus(uploaded.getId(), FileStatus.STATUS_FINISHED);
+    }
+
+    private void modifyStatus(String id, String status) {
+        Query query = new Query();
+        query.addCriteria(Criteria.where("_id").is(id));
+        Update update = new Update();
+        update.set("status", status);
+        mongoTemplate.updateFirst(query, update, "logical_file");
+    }
+
+
+    @Async
+    Future<Long> countTotalByModelId(String modelId) {
+        Long total = mongoTemplate.count(new Query(Criteria.where("modelId").is(modelId)), LogicalFile.class);
+        return new AsyncResult<>(total);
+    }
+
+    @Async
+    void removePhysicalFile(String id) {
+        fsOperations.delete(new Query(Criteria.where("_id").is(id)));
+    }
+
+    @Async
+    void removeDataByFileId(String fileId) {
+        mongoTemplate.remove(new Query(Criteria.where("fileId").is(fileId)), DataMap.class);
+    }
+}

src/main/java/ieven/server/webapp/domain/file/LogicalFile.java

@@ -16,5 +16,6 @@ public class LogicalFile {
     private ObjectId gridId;
     private String status;
     private String filename;
+    private Boolean generated;
     private String uploadDate;
 }

src/main/java/ieven/server/webapp/infrastructure/ThreadPoolTaskConfig.java

@@ -0,0 +1,55 @@
+package ieven.server.webapp.infrastructure;
+
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
+
+import java.util.concurrent.ThreadPoolExecutor;
+
+@Configuration
+public class ThreadPoolTaskConfig {
+
+/**
+ *   默认情况下，在创建了线程池后，线程池中的线程数为0，当有任务来之后，就会创建一个线程去执行任务，
+ *    当线程池中的线程数目达到corePoolSize后，就会把到达的任务放到缓存队列当中；
+ *  当队列满了，就继续创建线程，当线程数量大于等于maxPoolSize后，开始使用拒绝策略拒绝
+ */
+
+    /**
+     * 核心线程数（默认线程数）
+     */
+    private static final int corePoolSize = 20;
+    /**
+     * 最大线程数
+     */
+    private static final int maxPoolSize = 100;
+    /**
+     * 允许线程空闲时间（单位：默认为秒）
+     */
+    private static final int keepAliveTime = 10;
+    /**
+     * 缓冲队列大小
+     */
+    private static final int queueCapacity = 200;
+    /**
+     * 线程池名前缀
+     */
+    private static final String threadNamePrefix = "Async-Service-";
+
+    @Bean("taskExecutor") // bean的名称，默认为首字母小写的方法名
+    public ThreadPoolTaskExecutor taskExecutor() {
+        ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
+        executor.setCorePoolSize(corePoolSize);
+        executor.setMaxPoolSize(maxPoolSize);
+        executor.setQueueCapacity(queueCapacity);
+        executor.setKeepAliveSeconds(keepAliveTime);
+        executor.setThreadNamePrefix(threadNamePrefix);
+        // 线程池对拒绝任务的处理策略
+        // CallerRunsPolicy：由调用线程（提交任务的线程）处理该任务
+        executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
+        // 初始化
+        executor.initialize();
+        return executor;
+    }
+
+}