From b1d0c9afb9b5dd310fcf959f11ed926c91c1dcb5 Mon Sep 17 00:00:00 2001 From: CMM <2198256324@qq.com> Date: Tue, 27 Aug 2024 16:17:09 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1=E6=89=A7?= =?UTF-8?q?=E8=A1=8C=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kqapi/scheduler/task/CheckMattersUrlTask.java | 92 ++++++++++-------- .../kqapi/scheduler/task/ItemSynTask.java | 108 +++++++++++---------- .../kqapi/scheduler/task/ItemSynTestTask.java | 5 +- .../ningdatech/kqapi/scheduler/task/SynTask.java | 61 ++++++------ kq-vas-api/src/main/resources/application-dev.yml | 4 + kq-vas-api/src/main/resources/application-prod.yml | 4 + 6 files changed, 150 insertions(+), 124 deletions(-) diff --git a/kq-vas-api/src/main/java/com/ningdatech/kqapi/scheduler/task/CheckMattersUrlTask.java b/kq-vas-api/src/main/java/com/ningdatech/kqapi/scheduler/task/CheckMattersUrlTask.java index 9f5820e..65ed3e0 100644 --- a/kq-vas-api/src/main/java/com/ningdatech/kqapi/scheduler/task/CheckMattersUrlTask.java +++ b/kq-vas-api/src/main/java/com/ningdatech/kqapi/scheduler/task/CheckMattersUrlTask.java @@ -47,6 +47,9 @@ public class CheckMattersUrlTask { @Value("${hostname}") public String HOST; + @Value("${task.switch.is-open}") + private boolean flag; + @Value("${spring.profiles.active}") public String active; @@ -64,52 +67,55 @@ public class CheckMattersUrlTask { // return; // } - StopWatch stopWatch = new StopWatch(); - stopWatch.start(); - - log.info("==========开始校检事项表里的链接 会不会404 如果404就隐藏"); - - List menus = menuService.list(Wrappers.lambdaQuery(KqZzsfwMenu.class) - .eq(KqZzsfwMenu::getHasUrl, 1) - .isNotNull(KqZzsfwMenu::getWebapplyurl)); - Integer errNum = 0; - List errUrls = Lists.newArrayList(); - - for (KqZzsfwMenu menu : menus) { - RestTemplate restTemplate = new RestTemplate(generateHttpRequestFactory()); - String url = "https://www.zjzwfw.gov.cn/jpaas-zjservice-server/open-api/item/getItemDetail"; - MultiValueMap map = new LinkedMultiValueMap<>(); - - String webapplyurl = menu.getWebapplyurl(); - webapplyurl = webapplyurl.replace("/#",""); - Map queryParams = getQueryParams(new URL(webapplyurl)); - String localInnerCode = queryParams.get("localInnerCode"); - log.info("localInnerCode :{}",localInnerCode); - map.add("localInnerCode", localInnerCode); - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED); - HttpEntity> formEntity = new HttpEntity(map, headers); - ResponseEntity response = restTemplate.postForEntity(url, formEntity, Item.class); - System.out.println(response.getBody()); - Item body = response.getBody(); - assert body != null; - Item.ItemData itemData = body.getData(); - //如果是空的 - if(Objects.isNull(itemData.getItem())){ - //隐藏 - menuService.update(Wrappers.lambdaUpdate(KqZzsfwMenu.class) - .eq(KqZzsfwMenu::getId,menu.getId()) - .set(KqZzsfwMenu::getHasUrl,0) - .set(KqZzsfwMenu::getWebapplyurl,null)); - errNum += 1; - errUrls.add(webapplyurl); + // 线上环境运行 + if (flag && HOST.equals(InetAddress.getLocalHost().getHostName())) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); + + log.info("==========开始校检事项表里的链接 会不会404 如果404就隐藏"); + + List menus = menuService.list(Wrappers.lambdaQuery(KqZzsfwMenu.class) + .eq(KqZzsfwMenu::getHasUrl, 1) + .isNotNull(KqZzsfwMenu::getWebapplyurl)); + Integer errNum = 0; + List errUrls = Lists.newArrayList(); + + for (KqZzsfwMenu menu : menus) { + RestTemplate restTemplate = new RestTemplate(generateHttpRequestFactory()); + String url = "https://www.zjzwfw.gov.cn/jpaas-zjservice-server/open-api/item/getItemDetail"; + MultiValueMap map = new LinkedMultiValueMap<>(); + + String webapplyurl = menu.getWebapplyurl(); + webapplyurl = webapplyurl.replace("/#", ""); + Map queryParams = getQueryParams(new URL(webapplyurl)); + String localInnerCode = queryParams.get("localInnerCode"); + log.info("localInnerCode :{}", localInnerCode); + map.add("localInnerCode", localInnerCode); + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED); + HttpEntity> formEntity = new HttpEntity(map, headers); + ResponseEntity response = restTemplate.postForEntity(url, formEntity, Item.class); + System.out.println(response.getBody()); + Item body = response.getBody(); + assert body != null; + Item.ItemData itemData = body.getData(); + //如果是空的 + if (Objects.isNull(itemData.getItem())) { + //隐藏 + menuService.update(Wrappers.lambdaUpdate(KqZzsfwMenu.class) + .eq(KqZzsfwMenu::getId, menu.getId()) + .set(KqZzsfwMenu::getHasUrl, 0) + .set(KqZzsfwMenu::getWebapplyurl, null)); + errNum += 1; + errUrls.add(webapplyurl); + } } - } - log.info("不可用的链接有 :{}条 ,详情:{}",errNum,errUrls); + log.info("不可用的链接有 :{}条 ,详情:{}", errNum, errUrls); - stopWatch.stop(); - log.info("校检事项表里的链接结束====={}s",stopWatch.getTotalTimeSeconds()); + stopWatch.stop(); + log.info("校检事项表里的链接结束====={}s", stopWatch.getTotalTimeSeconds()); + } } public static void main(String[] args) throws Exception { diff --git a/kq-vas-api/src/main/java/com/ningdatech/kqapi/scheduler/task/ItemSynTask.java b/kq-vas-api/src/main/java/com/ningdatech/kqapi/scheduler/task/ItemSynTask.java index e1ae5dd..506bf08 100644 --- a/kq-vas-api/src/main/java/com/ningdatech/kqapi/scheduler/task/ItemSynTask.java +++ b/kq-vas-api/src/main/java/com/ningdatech/kqapi/scheduler/task/ItemSynTask.java @@ -1,5 +1,7 @@ package com.ningdatech.kqapi.scheduler.task; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.time.LocalDate; import java.time.LocalDateTime; import java.util.List; @@ -41,6 +43,9 @@ public class ItemSynTask { @Value("${hostname}") public String HOST; + @Value("${task.switch.is-open}") + private boolean flag; + private final IDscSxAdsShareItemQltQlsxCommonIDVKqService kqService; private final IrsManage irsManage; private final CachePlusOps cachePlusOps; @@ -51,63 +56,66 @@ public class ItemSynTask { * 每天凌晨5点同步数据 */ @Scheduled(cron = "0 0 5 * * ?") - public void synData() { - log.info("事项数据 ===================== 同步开始"); - StopWatch stopWatch = new StopWatch(); - stopWatch.start(); - // 每天凌晨5点循环拉取较前一天新更新的数据 - // 根据事项表中最后更新日期 循环更新至前一天的事项数据 - // 默认一页返回 - int pageNum = 1; - // irs接口最大返回数据量1000条 - int pageSize = 1000; + public void synData() throws UnknownHostException { + // 线上环境运行 + if (flag && HOST.equals(InetAddress.getLocalHost().getHostName())) { + log.info("事项数据 ===================== 同步开始"); + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); + // 每天凌晨5点循环拉取较前一天新更新的数据 + // 根据事项表中最后更新日期 循环更新至前一天的事项数据 + // 默认一页返回 + int pageNum = 1; + // irs接口最大返回数据量1000条 + int pageSize = 1000; - List records = Lists.newArrayList(); + List records = Lists.newArrayList(); - // 从缓存中取上次同步数据中最新的更新日期 - CacheKey key = new CacheKey(ITEM_BASIC_INFO_KEY); - String start = cachePlusOps.get(key); - // 如果缓存中不存在 从表中查询 - if (StringUtils.isBlank(start)){ - start = kqService.getNewestUpdateDate(); - } - String end = NdDateUtils.format(LocalDateTime.now().minusDays(1), NdDateUtils.DEFAULT_DATE_FORMAT); - LocalDate startDate = LocalDate.parse(start).plusDays(1); - LocalDate endDate = LocalDate.parse(end); - for (LocalDate date = startDate; !date.isAfter(endDate); date = date.plusDays(1)) { - boolean flag = true; - while (flag) { - String startDateStr = NdDateUtils.format(date, NdDateUtils.DEFAULT_DATE_FORMAT); - String endDateStr = NdDateUtils.format(date.plusDays(1), NdDateUtils.DEFAULT_DATE_FORMAT); - Object listObject = irsManage.itemBasicInfoList(startDateStr, endDateStr, pageSize, pageNum); - // 如果数据正常返回 - if (listObject instanceof JSONArray) { - JSONArray listJsonArray = (JSONArray) listObject; - log.info("事项列表数量为：{}", listJsonArray.size()); - getSaveRecords(listJsonArray, records); - // 如果listJsonArray数据量等于1000 需要继续拉取数据 - if (listJsonArray.size() == pageSize) { - pageNum++; + // 从缓存中取上次同步数据中最新的更新日期 + CacheKey key = new CacheKey(ITEM_BASIC_INFO_KEY); + String start = cachePlusOps.get(key); + // 如果缓存中不存在 从表中查询 + if (StringUtils.isBlank(start)) { + start = kqService.getNewestUpdateDate(); + } + String end = NdDateUtils.format(LocalDateTime.now().minusDays(1), NdDateUtils.DEFAULT_DATE_FORMAT); + LocalDate startDate = LocalDate.parse(start).plusDays(1); + LocalDate endDate = LocalDate.parse(end); + for (LocalDate date = startDate; !date.isAfter(endDate); date = date.plusDays(1)) { + boolean flag = true; + while (flag) { + String startDateStr = NdDateUtils.format(date, NdDateUtils.DEFAULT_DATE_FORMAT); + String endDateStr = NdDateUtils.format(date.plusDays(1), NdDateUtils.DEFAULT_DATE_FORMAT); + Object listObject = irsManage.itemBasicInfoList(startDateStr, endDateStr, pageSize, pageNum); + // 如果数据正常返回 + if (listObject instanceof JSONArray) { + JSONArray listJsonArray = (JSONArray) listObject; + log.info("事项列表数量为：{}", listJsonArray.size()); + getSaveRecords(listJsonArray, records); + // 如果listJsonArray数据量等于1000 需要继续拉取数据 + if (listJsonArray.size() == pageSize) { + pageNum++; + } else { + flag = false; + } } else { - flag = false; + log.error("事项列表数据异常：{}", listObject.toString()); + return; } - } else { - log.error("事项列表数据异常：{}", listObject.toString()); - return; } } + log.info("{} 数据列表数量为：{}", startDate, records.size()); + kqService.saveBatch(records); + log.info("事项同步成功，更新时间：{}", startDate); + // 更新缓存中的更新时间 + // 获取此次保存记录中的最新更新时间 + records.sort((o1, o2) -> o2.getUpdateDate().compareTo(o1.getUpdateDate())); + LocalDateTime updateDate = records.get(0).getUpdateDate(); + String updateDateStr = NdDateUtils.format(updateDate, NdDateUtils.DEFAULT_DATE_FORMAT); + cachePlusOps.set(key, updateDateStr); + stopWatch.stop(); + log.info("事项数据 ===================== 同步总共耗时 :{} s", stopWatch.getTotalTimeSeconds()); } - log.info("{} 数据列表数量为：{}", startDate, records.size()); - kqService.saveBatch(records); - log.info("事项同步成功，更新时间：{}", startDate); - // 更新缓存中的更新时间 - // 获取此次保存记录中的最新更新时间 - records.sort((o1, o2) -> o2.getUpdateDate().compareTo(o1.getUpdateDate())); - LocalDateTime updateDate = records.get(0).getUpdateDate(); - String updateDateStr = NdDateUtils.format(updateDate, NdDateUtils.DEFAULT_DATE_FORMAT); - cachePlusOps.set(key, updateDateStr); - stopWatch.stop(); - log.info("事项数据 ===================== 同步总共耗时 :{} s", stopWatch.getTotalTimeSeconds()); } private void getSaveRecords(JSONArray listJsonArray, List records) { diff --git a/kq-vas-api/src/main/java/com/ningdatech/kqapi/scheduler/task/ItemSynTestTask.java b/kq-vas-api/src/main/java/com/ningdatech/kqapi/scheduler/task/ItemSynTestTask.java index eefd543..d7e7bad 100644 --- a/kq-vas-api/src/main/java/com/ningdatech/kqapi/scheduler/task/ItemSynTestTask.java +++ b/kq-vas-api/src/main/java/com/ningdatech/kqapi/scheduler/task/ItemSynTestTask.java @@ -50,10 +50,7 @@ public class ItemSynTestTask { String transmitListUrl = "https://qyxspt.kq.gov.cn:33060/kq/api/v1/irs/item-basic-info-list"; String transmitDetailUrl = "https://qyxspt.kq.gov.cn:33060/kq/api/v1/irs/item-basic-info-detail"; - /** - * 每天凌晨5点同步数据 - */ - //@Scheduled(cron = "0 0 5 * * ?") + public void synData() { log.info("事项数据 ===================== 同步开始"); StopWatch stopWatch = new StopWatch(); diff --git a/kq-vas-api/src/main/java/com/ningdatech/kqapi/scheduler/task/SynTask.java b/kq-vas-api/src/main/java/com/ningdatech/kqapi/scheduler/task/SynTask.java index b860a77..0c8362c 100644 --- a/kq-vas-api/src/main/java/com/ningdatech/kqapi/scheduler/task/SynTask.java +++ b/kq-vas-api/src/main/java/com/ningdatech/kqapi/scheduler/task/SynTask.java @@ -21,6 +21,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; +import java.net.InetAddress; import java.net.UnknownHostException; import java.time.LocalDateTime; import java.util.ArrayList; @@ -40,6 +41,9 @@ public class SynTask { @Value("${hostname}") public String HOST; + @Value("${task.switch.is-open}") + private boolean flag; + @Value("${spring.profiles.active}") public String active; @@ -70,36 +74,39 @@ public class SynTask { */ @Scheduled(cron = "0 30 8 * * ?") public void synData() throws UnknownHostException { - log.info("政务数据 ===================== 同步开始"); - StopWatch stopWatch = new StopWatch(); - stopWatch.start(); - - //1.处理 去重数据 - if (matterCacheHelper.refreshAll()) { - matterManage.removeAllDup(); - List allDup = matterCacheHelper.all(); - matterDeduplicateService.saveBatch(allDup); - - //2. 事项 - List menus = menuService.list(); - for (KqZzsfwMenu menu : menus) { - String url = matterCacheHelper.getUrl(menu.getItemName()); - if (StringUtils.isBlank(url)) { - menuService.update(Wrappers.lambdaUpdate(KqZzsfwMenu.class) - .eq(KqZzsfwMenu::getId, menu.getId()) - .set(KqZzsfwMenu::getWebapplyurl, null) - .set(KqZzsfwMenu::getHasUrl, CommonEnum.NO.getCode())); - } else { - menuService.update(Wrappers.lambdaUpdate(KqZzsfwMenu.class) - .eq(KqZzsfwMenu::getId, menu.getId()) - .set(KqZzsfwMenu::getWebapplyurl, url) - .set(KqZzsfwMenu::getHasUrl, CommonEnum.YES.getCode())); + // 线上环境运行 + if (flag && HOST.equals(InetAddress.getLocalHost().getHostName())) { + log.info("政务数据 ===================== 同步开始"); + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); + + //1.处理 去重数据 + if (matterCacheHelper.refreshAll()) { + matterManage.removeAllDup(); + List allDup = matterCacheHelper.all(); + matterDeduplicateService.saveBatch(allDup); + + //2. 事项 + List menus = menuService.list(); + for (KqZzsfwMenu menu : menus) { + String url = matterCacheHelper.getUrl(menu.getItemName()); + if (StringUtils.isBlank(url)) { + menuService.update(Wrappers.lambdaUpdate(KqZzsfwMenu.class) + .eq(KqZzsfwMenu::getId, menu.getId()) + .set(KqZzsfwMenu::getWebapplyurl, null) + .set(KqZzsfwMenu::getHasUrl, CommonEnum.NO.getCode())); + } else { + menuService.update(Wrappers.lambdaUpdate(KqZzsfwMenu.class) + .eq(KqZzsfwMenu::getId, menu.getId()) + .set(KqZzsfwMenu::getWebapplyurl, url) + .set(KqZzsfwMenu::getHasUrl, CommonEnum.YES.getCode())); + } } } - } - stopWatch.stop(); - log.info("政务数据 ===================== 同步总共耗时 :{} s", stopWatch.getTotalTimeSeconds()); + stopWatch.stop(); + log.info("政务数据 ===================== 同步总共耗时 :{} s", stopWatch.getTotalTimeSeconds()); + } } public void synMenu() { diff --git a/kq-vas-api/src/main/resources/application-dev.yml b/kq-vas-api/src/main/resources/application-dev.yml index d932e37..4825516 100644 --- a/kq-vas-api/src/main/resources/application-dev.yml +++ b/kq-vas-api/src/main/resources/application-dev.yml @@ -130,7 +130,11 @@ swagger: sys: web-url: http://kqzh.ningdatech.com +# 定时任务配置 hostname: iZbp13nwyvib53j4j1p2xoZ +task: + switch: + is-open: false #开关 jasypt: encryptor: password: CodeSheep diff --git a/kq-vas-api/src/main/resources/application-prod.yml b/kq-vas-api/src/main/resources/application-prod.yml index 52b8d48..96899aa 100644 --- a/kq-vas-api/src/main/resources/application-prod.yml +++ b/kq-vas-api/src/main/resources/application-prod.yml @@ -124,7 +124,11 @@ log: swagger: enabled: false +# 定时任务配置 hostname: iZr3i038ar3e2l7qn9ji2oZ +task: + switch: + is-open: true #开关 sys: web-url: https://qyxspt.kq.gov.cn:33060