|
@@ -45,7 +45,7 @@ public class DataStaticAnalysisTotal extends Ops implements Runnable {
|
|
|
log.info("数据分析-主体信息汇总开始");
|
|
|
// 直接删除并创建表
|
|
|
String newFileId = "";
|
|
|
- String newFileIdAll = "";
|
|
|
+ //String newFileIdAll = "";
|
|
|
String dateFileId = getFileId("数据分析-主体信息汇总明细", modelId, mongoTemplate);
|
|
|
String cft = getFileId(TenpayHeaders.getGeneratedName(3), modelId, mongoTemplate);
|
|
|
String wld = getFileId(AlipayHeaders.getGeneratedName(2), modelId, mongoTemplate);
|
|
@@ -56,21 +56,21 @@ public class DataStaticAnalysisTotal extends Ops implements Runnable {
|
|
|
newFileId = createLogicAndDeleteOld("数据分析-主体信息汇总", modelId, headerProperties.getStaticUserOutput(), mongoTemplate, fileService);
|
|
|
insertFromTo(dateFileId, newFileId, modelId, mongoTemplate);
|
|
|
|
|
|
- //维护一张人员信息基础表,用于模型计算
|
|
|
+ /* //维护一张人员信息基础表,用于模型计算
|
|
|
newFileIdAll = createLogicAndDeleteOld("数据分析-所有主体信息汇总", modelId, headerProperties.getStaticUserAllOutput(), mongoTemplate, fileService);
|
|
|
- insertFromTo(dateFileId, newFileIdAll, modelId, mongoTemplate);
|
|
|
+ insertFromTo(dateFileId, newFileIdAll, modelId, mongoTemplate);*/
|
|
|
|
|
|
String newFileIdAll1 = createLogicAndDeleteOld("数据分析-共同关系网络", modelId, headerProperties.getStaticUserAllOutput(), mongoTemplate, fileService);
|
|
|
insertFromTo(dateFileId, newFileIdAll1, modelId, mongoTemplate);
|
|
|
}
|
|
|
Set<String> alipayIds = new HashSet<>();
|
|
|
- Set<String> alipayIdsAll = new HashSet<>();
|
|
|
+ //Set<String> alipayIdsAll = new HashSet<>();
|
|
|
Map<String, DataMap> idToData = new HashMap<>();
|
|
|
Map<String, Set<String>> idToNames = new HashMap<>();
|
|
|
Document origin;
|
|
|
List<DataMap> needToSave = new ArrayList<>(1000);
|
|
|
- List<DataMap> needToSaveall = new ArrayList<>(1000);
|
|
|
- Map<String, Object> dataMapUser = new HashMap<>();
|
|
|
+ //List<DataMap> needToSaveall = new ArrayList<>(1000);
|
|
|
+ //Map<String, Object> dataMapUser = new HashMap<>();
|
|
|
while (cursor.hasNext()) {
|
|
|
try {
|
|
|
origin = cursor.next();
|
|
@@ -106,7 +106,7 @@ public class DataStaticAnalysisTotal extends Ops implements Runnable {
|
|
|
}
|
|
|
namesSet.add(alipayUsername);
|
|
|
}
|
|
|
- getDataAll(needToSaveall, origin, alipayIdsAll, newFileIdAll, dataMapUser, idToData, dateFileId2);
|
|
|
+ /* getDataAll(needToSaveall, origin, alipayIdsAll, newFileIdAll, dataMapUser, idToData, dateFileId2);
|
|
|
if (needToSaveall.size() >= 1000) {
|
|
|
List<DataMap> newDataList = new ArrayList<>();
|
|
|
for (DataMap map : needToSaveall) {
|
|
@@ -130,7 +130,7 @@ public class DataStaticAnalysisTotal extends Ops implements Runnable {
|
|
|
saveLines(newDataList, mongoTemplate);
|
|
|
}
|
|
|
needToSaveall.clear();
|
|
|
- }
|
|
|
+ }*/
|
|
|
|
|
|
} catch (Exception exception) {
|
|
|
log.error(exception.getMessage(), exception);
|
|
@@ -156,7 +156,7 @@ public class DataStaticAnalysisTotal extends Ops implements Runnable {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- if (CollectionUtils.isNotEmpty(needToSaveall)) {
|
|
|
+ /*if (CollectionUtils.isNotEmpty(needToSaveall)) {
|
|
|
List<DataMap> newDataList = new ArrayList<>();
|
|
|
for (DataMap map : needToSaveall) {
|
|
|
String zh = String.valueOf(map.get("账号"));
|
|
@@ -178,7 +178,7 @@ public class DataStaticAnalysisTotal extends Ops implements Runnable {
|
|
|
if (CollectionUtils.isNotEmpty(newDataList)) {
|
|
|
saveLines(newDataList, mongoTemplate);
|
|
|
}
|
|
|
- }
|
|
|
+ }*/
|
|
|
}
|
|
|
latch.countDown();
|
|
|
log.info("数据分析-主体信息汇总开始");
|
|
@@ -196,9 +196,7 @@ public class DataStaticAnalysisTotal extends Ops implements Runnable {
|
|
|
//账户状态", "账号", "注册姓名", "注册时间", "注册身份证号", "绑定手机", "绑定状态", "开户行信息", "银行账号"
|
|
|
if (StringUtils.isNotBlank(dateFileId2)) {
|
|
|
List<DataMap> dataMaps = this.mongoTemplate.find((new Query(
|
|
|
- Criteria.where("modelId")
|
|
|
- .is(modelId)
|
|
|
- .and("fileId")
|
|
|
+ Criteria.where("fileId")
|
|
|
.is(dateFileId2)
|
|
|
.and("账号")
|
|
|
.is(account)
|
|
@@ -215,9 +213,7 @@ public class DataStaticAnalysisTotal extends Ops implements Runnable {
|
|
|
// "用户ID", "登录邮箱", "登录手机", "账户名称", "证件类型", "证件号", "可用余额", "注册时间", "绑定手机", "绑定银行卡", "对应的协查数据"));
|
|
|
if (StringUtils.isNotBlank(dateFileId2)) {
|
|
|
List<DataMap> dataMaps = this.mongoTemplate.find((new Query(
|
|
|
- Criteria.where("modelId")
|
|
|
- .is(modelId)
|
|
|
- .and("fileId")
|
|
|
+ Criteria.where("fileId")
|
|
|
.is(dateFileId2)
|
|
|
.and("用户ID")
|
|
|
.is(account))), DataMap.class);
|
|
@@ -276,37 +272,42 @@ public class DataStaticAnalysisTotal extends Ops implements Runnable {
|
|
|
//账户状态", "账号", "注册姓名", "注册时间", "注册身份证号", "绑定手机", "绑定状态", "开户行信息", "银行账号"
|
|
|
if (CollectionUtils.isNotEmpty(accountList)) {
|
|
|
for (String s : accountList) {
|
|
|
- DataMap dataMap = havedUser.get(s);
|
|
|
+ String s1 = s.split("-")[0];
|
|
|
+ DataMap dataMap = havedUser.get(s1);
|
|
|
if (dataMap != null) {
|
|
|
DataMap newDataMap = new DataMap();
|
|
|
newDataMap.put("证件号", String.valueOf(dataMap.get("证件号码")));
|
|
|
- newDataMap.put("账户名称", String.valueOf(dataMap.get("交易主体户名")));
|
|
|
+ newDataMap.put("账户名称", PublicStatic.checkStr(String.valueOf(dataMap.get("交易主体户名")))?s.split("-")[1]:String.valueOf(dataMap.get("交易主体户名")));
|
|
|
newDataMap.put("账号", String.valueOf(dataMap.get("交易主体账号")));
|
|
|
dataMaps.add(newDataMap);
|
|
|
} else {
|
|
|
if (StringUtils.isNotBlank(dateFileId2)) {
|
|
|
List<DataMap> dataMapnew = this.mongoTemplate.find((new Query(
|
|
|
- Criteria.where("modelId")
|
|
|
- .is(modelId)
|
|
|
- .and("fileId")
|
|
|
+ Criteria.where("fileId")
|
|
|
.is(dateFileId2)
|
|
|
.and("账号")
|
|
|
- .in(s))), DataMap.class);
|
|
|
+ .in(s1))), DataMap.class);
|
|
|
|
|
|
if (CollectionUtils.isNotEmpty(dataMapnew)) {
|
|
|
for (DataMap map : dataMapnew) {
|
|
|
DataMap newDataMap = new DataMap();
|
|
|
newDataMap.put("证件号", String.valueOf(map.get("注册身份证号")));
|
|
|
- newDataMap.put("账户名称", String.valueOf(map.get("注册姓名")));
|
|
|
+ newDataMap.put("账户名称", PublicStatic.checkStr(String.valueOf(map.get("注册姓名")))?s.split("-")[1]:String.valueOf(map.get("注册姓名")));
|
|
|
newDataMap.put("账号", String.valueOf(map.get("账号")));
|
|
|
dataMaps.add(newDataMap);
|
|
|
}
|
|
|
+ }else {
|
|
|
+ DataMap newDataMap = new DataMap();
|
|
|
+ newDataMap.put("证件号", "");
|
|
|
+ newDataMap.put("账户名称", StringUtils.isBlank(String.valueOf(dataMaphm.get(s))) || "null".equals(String.valueOf(dataMaphm.get(s))) ? "" : String.valueOf(dataMaphm.get(s)));
|
|
|
+ newDataMap.put("账号", s1);
|
|
|
+ dataMaps.add(newDataMap);
|
|
|
}
|
|
|
} else {
|
|
|
DataMap newDataMap = new DataMap();
|
|
|
newDataMap.put("证件号", "");
|
|
|
newDataMap.put("账户名称", StringUtils.isBlank(String.valueOf(dataMaphm.get(s))) || "null".equals(String.valueOf(dataMaphm.get(s))) ? "" : String.valueOf(dataMaphm.get(s)));
|
|
|
- newDataMap.put("账号", s.split("-")[0]);
|
|
|
+ newDataMap.put("账号", s1);
|
|
|
dataMaps.add(newDataMap);
|
|
|
}
|
|
|
}
|
|
@@ -345,7 +346,7 @@ public class DataStaticAnalysisTotal extends Ops implements Runnable {
|
|
|
newDataMap.put("fileId", newFileIdAll);
|
|
|
newDataMap.put("证件号", "");
|
|
|
newDataMap.put("账户名称", StringUtils.isBlank(String.valueOf(dataMaphm.get(s))) || "null".equals(String.valueOf(dataMaphm.get(s))) ? "" : String.valueOf(dataMaphm.get(s)));
|
|
|
- newDataMap.put("账号", s);
|
|
|
+ newDataMap.put("账号", s.split("-")[0]);
|
|
|
needToSaveall.add(newDataMap);
|
|
|
alipayIdsAll.add(s);
|
|
|
}
|
|
@@ -356,52 +357,55 @@ public class DataStaticAnalysisTotal extends Ops implements Runnable {
|
|
|
//账户状态", "账号", "注册姓名", "注册时间", "注册身份证号", "绑定手机", "绑定状态", "开户行信息", "银行账号"
|
|
|
if (CollectionUtils.isNotEmpty(accountList)) {
|
|
|
for (String s : accountList) {
|
|
|
- DataMap dataMap = havedUser.get(s);
|
|
|
+ String s1 = s.split("-")[0];
|
|
|
+ DataMap dataMap = havedUser.get(s1);
|
|
|
if (dataMap != null) {
|
|
|
DataMap newDataMap = new DataMap();
|
|
|
newDataMap.put("证件号", String.valueOf(dataMap.get("证件号码")));
|
|
|
- newDataMap.put("账户名称", String.valueOf(dataMap.get("交易主体户名")));
|
|
|
+ newDataMap.put("账户名称", PublicStatic.checkStr(String.valueOf(dataMap.get("交易主体户名")))? s.split("-")[1]:String.valueOf(dataMap.get("交易主体户名")));
|
|
|
newDataMap.put("账号", String.valueOf(dataMap.get("交易主体账号")));
|
|
|
dataMaps.add(newDataMap);
|
|
|
} else {
|
|
|
if (StringUtils.isNotBlank(dateFileId2)) {
|
|
|
List<DataMap> dataMapnew = this.mongoTemplate.find((new Query(
|
|
|
- Criteria.where("modelId")
|
|
|
- .is(modelId)
|
|
|
- .and("fileId")
|
|
|
+ Criteria.where("fileId")
|
|
|
.is(dateFileId2)
|
|
|
.and("用户ID")
|
|
|
- .in(accountList))), DataMap.class);
|
|
|
+ .in(s1))), DataMap.class);
|
|
|
if (CollectionUtils.isNotEmpty(dataMapnew)) {
|
|
|
for (DataMap map : dataMapnew) {
|
|
|
DataMap newDataMap = new DataMap();
|
|
|
newDataMap.put("证件号", String.valueOf(map.get("证件号")));
|
|
|
- newDataMap.put("账户名称", String.valueOf(map.get("账户名称")));
|
|
|
+ newDataMap.put("账户名称", PublicStatic.checkStr(String.valueOf(dataMap.get("交易主体户名")))? s.split("-")[1]:String.valueOf(dataMap.get("交易主体户名")));
|
|
|
newDataMap.put("账号", String.valueOf(map.get("用户ID")));
|
|
|
dataMaps.add(newDataMap);
|
|
|
}
|
|
|
+ }else {
|
|
|
+ DataMap newDataMap = new DataMap();
|
|
|
+ newDataMap.put("证件号", "");
|
|
|
+ newDataMap.put("账户名称", StringUtils.isBlank(String.valueOf(dataMaphm.get(s))) || "null".equals(String.valueOf(dataMaphm.get(s))) ? "" : String.valueOf(dataMaphm.get(s)));
|
|
|
+ newDataMap.put("账号", s1);
|
|
|
+ dataMaps.add(newDataMap);
|
|
|
}
|
|
|
-
|
|
|
} else {
|
|
|
DataMap newDataMap = new DataMap();
|
|
|
- newDataMap.put("fileId", newFileIdAll);
|
|
|
newDataMap.put("证件号", "");
|
|
|
newDataMap.put("账户名称", StringUtils.isBlank(String.valueOf(dataMaphm.get(s))) || "null".equals(String.valueOf(dataMaphm.get(s))) ? "" : String.valueOf(dataMaphm.get(s)));
|
|
|
- newDataMap.put("账号", s);
|
|
|
+ newDataMap.put("账号", s1);
|
|
|
dataMaps.add(newDataMap);
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
if (CollectionUtils.isNotEmpty(dataMaps)) {
|
|
|
for (DataMap dataMap : dataMaps) {
|
|
|
- if (!PublicStatic.checkStr(String.valueOf(dataMap.get("用户ID"))) && !alipayIdsAll.contains(String.valueOf(dataMap.get("用户ID")))) {
|
|
|
+ if (!PublicStatic.checkStr(String.valueOf(dataMap.get("账号"))) && !alipayIdsAll.contains(String.valueOf(dataMap.get("账号")))) {
|
|
|
DataMap newDataMap = new DataMap();
|
|
|
newDataMap.put("fileId", newFileIdAll);
|
|
|
newDataMap.put("证件号", String.valueOf(dataMap.get("证件号")));
|
|
|
newDataMap.put("账户名称", String.valueOf(dataMap.get("账户名称")));
|
|
|
newDataMap.put("账号", String.valueOf(dataMap.get("账号")));
|
|
|
needToSaveall.add(newDataMap);
|
|
|
- alipayIdsAll.add(String.valueOf(dataMap.get("用户ID")));
|
|
|
+ alipayIdsAll.add(String.valueOf(dataMap.get("账号")));
|
|
|
}
|
|
|
}
|
|
|
} else {
|
|
@@ -411,7 +415,7 @@ public class DataStaticAnalysisTotal extends Ops implements Runnable {
|
|
|
newDataMap.put("fileId", newFileIdAll);
|
|
|
newDataMap.put("证件号", "");
|
|
|
newDataMap.put("账户名称", StringUtils.isBlank(String.valueOf(dataMaphm.get(s))) || "null".equals(String.valueOf(dataMaphm.get(s))) ? "" : String.valueOf(dataMaphm.get(s)));
|
|
|
- newDataMap.put("账号", s);
|
|
|
+ newDataMap.put("账号", s.split("-")[0]);
|
|
|
needToSaveall.add(newDataMap);
|
|
|
alipayIdsAll.add(s);
|
|
|
}
|