package ieven.server.webapp.domain.file; import com.mongodb.client.gridfs.model.GridFSFile; import com.mongodb.client.result.DeleteResult; import ieven.server.webapp.domain.IdInput; import ieven.server.webapp.domain.data.DataMap; import ieven.server.webapp.domain.model.Model; import ieven.server.webapp.infrastructure.wrapper.Mapped; import ieven.server.webapp.service.impl.MongoExcelServiceImpl; import ieven.server.webapp.util.excel.ExcelXlsReader; import ieven.server.webapp.util.excel.ExcelXlsxReader; import ieven.server.webapp.util.excel.PublicStatic; import lombok.extern.slf4j.Slf4j; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; import org.apache.commons.lang3.StringUtils; import org.apache.poi.openxml4j.exceptions.OpenXML4JException; import org.bson.types.ObjectId; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Lazy; import org.springframework.data.domain.Sort; import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.data.mongodb.core.query.Criteria; import org.springframework.data.mongodb.core.query.Query; import org.springframework.data.mongodb.core.query.Update; import org.springframework.data.mongodb.gridfs.GridFsOperations; import org.springframework.data.mongodb.gridfs.GridFsResource; import org.springframework.data.mongodb.gridfs.GridFsTemplate; import org.springframework.scheduling.annotation.Async; import org.springframework.scheduling.annotation.AsyncResult; import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; import org.xml.sax.SAXException; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.text.SimpleDateFormat; import java.util.Date; import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.ThreadPoolExecutor; import java.util.stream.Collectors; @Slf4j @Service @EnableAsync public class FileService { @Autowired private GridFsTemplate gridFsTemplate; @Autowired private MongoTemplate mongoTemplate; @Autowired private GridFsOperations fsOperations; @Autowired @Lazy private FileService fileService; public Mapped storeUploaded(MultipartFile file, String modelId) { String filename = file.getOriginalFilename(); String contentType = file.getContentType(); //直接上传到mongo ObjectId objectId = null; try { objectId = gridFsTemplate.store(file.getInputStream(), filename, contentType); } catch (IOException e) { log.info("转储文件失败"); e.printStackTrace(); return Mapped.ERROR("转储文件失败"); } LogicalFile logicalFile = new LogicalFile(); logicalFile.setFilename(filename); logicalFile.setUploadDate(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date())); logicalFile.setModelId(modelId); logicalFile.setGridId(objectId); logicalFile.setGenerated(Boolean.FALSE); logicalFile.setStatus(FileStatus.STATUS_LOADING); //使用主键自动生成id logicalFile = mongoTemplate.insert(logicalFile); //异步读取数据并存储到mongo fileService.readFile(logicalFile); // fileService.readFileWhileStoreDataAsync(objectId); return Mapped.OK(); } @Async public void readFile(LogicalFile uploaded) { GridFSFile fsFile = fsOperations.findOne(new Query(Criteria.where("_id").is(uploaded.getGridId()))); if (fsFile == null) { //解析失败 modifyStatus(uploaded.getId(), FileStatus.STATUS_ERROR); return; } GridFsResource resource = fsOperations.getResource(fsFile); String filename = resource.getFilename(); if (StringUtils.isBlank(filename)) { //解析失败 modifyStatus(uploaded.getId(), FileStatus.STATUS_ERROR); return; } try { InputStream inputStream = resource.getInputStream(); if (filename.endsWith(".xlsx")) { MongoExcelServiceImpl mongoExcelService = new MongoExcelServiceImpl(mongoTemplate, uploaded.getId()); ExcelXlsxReader reader = new ExcelXlsxReader(mongoExcelService); reader.processStream(inputStream); mongoExcelService.insertRest(); } else if (filename.endsWith(".xls")) { MongoExcelServiceImpl mongoExcelService = new MongoExcelServiceImpl(mongoTemplate, uploaded.getId()); ExcelXlsReader reader = new ExcelXlsReader(mongoExcelService); reader.processStream(inputStream); mongoExcelService.insertRest(); } else if (filename.endsWith(".csv")) { MongoExcelServiceImpl mongoExcelService = new MongoExcelServiceImpl(mongoTemplate, uploaded.getId()); CSVParser csvParser = CSVFormat.EXCEL.parse(new InputStreamReader(inputStream, "gbk")); int curRow = 0; for (CSVRecord record : csvParser) { mongoExcelService.getRows(0, curRow, record.stream().map(PublicStatic::removeAllSpecial).collect(Collectors.toList())); curRow++; } mongoExcelService.insertRest(); } } catch (IOException | OpenXML4JException | SAXException e) { //解析失败 modifyStatus(uploaded.getId(), FileStatus.STATUS_ERROR); e.printStackTrace(); } //解析成功 modifyStatus(uploaded.getId(), FileStatus.STATUS_FINISHED); } private void modifyStatus(String id, String status) { Query query = new Query(); query.addCriteria(Criteria.where("_id").is(id)); Update update = new Update(); update.set("status", status); mongoTemplate.updateFirst(query, update, "logical_file"); } public Mapped listUploaded(FileListInput fileListInput) { String modelId = fileListInput.getModelId(); if (modelId == null) { return Mapped.ERROR("没有选择任何分组"); } Future longFuture = fileService.countTotalByModelId(modelId); Future modelNameFuture = fileService.retrieveModelname(modelId); int page = fileListInput.getPage(); int pageSize = fileListInput.getPageSize(); int skip = (page - 1) * pageSize; Query query = new Query(Criteria.where("modelId").is(modelId)); query.with(Sort.by( Sort.Order.desc("uploadDate") )); List resultList = mongoTemplate.find( query.skip(skip).limit(pageSize), LogicalFile.class); Long total = null; String modelName = ""; try { total = longFuture.get(); modelName = modelNameFuture.get(); } catch (InterruptedException | ExecutionException e) { return Mapped.ERROR("查询出错"); } int i = 0; for (LogicalFile file : resultList) { file.setModelName(modelName); file.setCount(++i); } Mapped res = Mapped.OK(); res.put("code", 200); res.put("result", resultList); res.put("total", total); res.put("currentPage", page); res.put("pageSize", pageSize); return res; } public Mapped deleteById(IdInput idInput) { String id = idInput.getId(); Query query = new Query(Criteria.where("id").is(id)); LogicalFile file = mongoTemplate.findOne(query, LogicalFile.class); //删除物理文件 if (file != null && file.getGridId() != null) { String physicalId = file.getGridId().toHexString(); fileService.removePhysicalFile(physicalId); } //删除数据 fileService.removeDataByFileId(id); DeleteResult result = mongoTemplate.remove(query, LogicalFile.class); if (result.getDeletedCount() > 0) { return Mapped.OK(); } else { return Mapped.ERROR("删除失败"); } } @Async Future countTotalByModelId(String modelId) { Long total = mongoTemplate.count(new Query(Criteria.where("modelId").is(modelId)), LogicalFile.class); return new AsyncResult<>(total); } @Async void removePhysicalFile(String id) { fsOperations.delete(new Query(Criteria.where("_id").is(id))); } @Async void removeDataByFileId(String fileId) { mongoTemplate.remove(new Query(Criteria.where("fileId").is(fileId)), DataMap.class); } @Async Future retrieveModelname(String modelId) { Query query = new Query(Criteria.where("id").is(modelId)); query.fields().include("modelName"); Model model = mongoTemplate.findOne(query, Model.class, "model"); return new AsyncResult<>(model != null ? model.getModelName() : ""); } }