分段同步

src/main/java/com/galaxis/manatee/task/ChuanyunHumanResourceScheduledTask.java

@@ -60,8 +60,7 @@ public class ChuanyunHumanResourceScheduledTask {
     /**
      * 每天3点更新日志记录
      */
-//    @Scheduled(cron = "0 0 1 * * *")
-    @Scheduled(fixedDelay = 3600000L)
+    @Scheduled(cron = "0 0 1 * * *")
     private void updateLog() {
         var objectMapper = new ObjectMapper();
         var start = 0;
@@ -71,6 +70,7 @@ public class ChuanyunHumanResourceScheduledTask {
         //由于需要统计流程中的日志，在日志数据同步时将状态为2的数据一并统计
 //        matchersString.add("Status_2,1");
         matchersString.add("Status_6,[1;2]");
+        matchersString.add("logDate_4,2020-08-01 00:00:00");
         while (flag) {
             Long startTime= Instant.now().getEpochSecond();
             try {
@@ -134,7 +134,7 @@ public class ChuanyunHumanResourceScheduledTask {
             }
             log.info(Instant.now().getEpochSecond()-startTime+"日志100条保存时间");
         }
-        log.info("日志信息保存" + totalCount);
+        log.info("历史日志信息保存" + totalCount);
     }
 
     /**
@@ -241,4 +241,80 @@ public class ChuanyunHumanResourceScheduledTask {
         }
         log.info("凯乐士薪酬项同步"+totalCount);
     }
+
+    @Scheduled(fixedDelay = 3600000L)
+    private void updateRecentLog() {
+        var objectMapper = new ObjectMapper();
+        var start = 0;
+        var totalCount = 0L;
+        var flag = true;
+        List<String> matchersString=new ArrayList<>();
+        //由于需要统计流程中的日志，在日志数据同步时将状态为2的数据一并统计
+        matchersString.add("Status_6,[1;2]");
+        matchersString.add("logDate_1,2020-08-01 00:00:00");
+        while (flag) {
+            Long startTime= Instant.now().getEpochSecond();
+            try {
+                //从氚云查询数据
+                var filter= Filter.instance(start,start+pageSize,true,Filter.AND,matchersString);
+                var chuanyunFindAllResponse = chuanYunManager.findAll(ChuanyunLogDO.SCHEMA_CODE, filter);
+                if (chuanyunFindAllResponse.getReturnData() == null) {
+                    flag = false;
+                    continue;
+                }
+                if (chuanyunFindAllResponse.getReturnData().getTotalCount() >= start + pageSize) {
+                    start += pageSize;
+                } else {
+                    flag = false;
+                }
+                //转化为POJO
+                List<ChuanyunLogDO> result = objectMapper.convertValue(chuanyunFindAllResponse.getReturnData().getBizObjectArray(), new TypeReference<>() {
+                });
+                result.forEach(chuanyunLogDO -> {
+                    if(chuanyunLogDO.getDayLogList()==null||chuanyunLogDO.getDayLogList().size()==0){
+                        chuanyunLogDO.setDayLogList(new ArrayList<>());
+                        log.warn(chuanyunLogDO+"");
+                        log.warn("异常数据");
+                    }else{
+                        //如果有重复就删除
+                        chuanyunLogDao.findById(chuanyunLogDO.getObjectId()).ifPresent(toDelete -> {
+//                            log.info("删除" + toDelete.getObjectId());
+                            chuanyunLogDao.delete(toDelete);
+                        });
+                        //根据projectId获取code,如果是生产项目，则不用处理
+                        //针对前期测试数据，可能出现日志为空的情况，跳过不处理
+                        chuanyunLogDO.getDayLogList().forEach(chuanyunDayLog -> {
+                            try{
+                                if(!"生产项目".equals(chuanyunDayLog.getProjectType())&& StringUtils.isEmpty(chuanyunDayLog.getProjectId())){
+                                    log.info(chuanyunLogDO+"");
+                                }else if(!"生产项目".equals(chuanyunDayLog.getProjectType())){
+                                    ChuanyunGroupProjectDO chuanyunGroupProjectDO=chuanyunGroupProjectDao.findById(chuanyunDayLog.getProjectId()).orElseThrow();
+                                    chuanyunDayLog.setProjectCode(chuanyunGroupProjectDO.getProjectCode());
+                                }
+                            }catch (Exception e){
+                                e.printStackTrace();
+                                log.warn(chuanyunDayLog+"");
+                            }
+                        });
+                        //判断所属部门是否正确
+                        //目前已经通过excel方式更新完成这个方法暂时不需要使用
+//                        checkDepartment(chuanyunLogDO);
+                    }
+                    //处理项目userId
+                    if(chuanyunLogDO.getUserObject()!=null){
+                        chuanyunLogDO.setUserId(chuanyunLogDO.getUserObject().getObjectId());
+                    }
+                });
+                //获取项目总数
+                totalCount = chuanyunFindAllResponse.getReturnData().getTotalCount();
+                //保存
+                chuanyunLogDao.saveAll(result);
+            } catch (Exception e) {
+                log.error(e.getMessage());
+                e.printStackTrace();
+            }
+            log.info(Instant.now().getEpochSecond()-startTime+"日志100条保存时间");
+        }
+        log.info("最近的日志日志信息保存" + totalCount);
+    }
 }