poffyzhang
/
kq-value-added-project
1
0
Fork 0
Code Issues 0 Pull Requests 0 Releases 0 Wiki Activity
Browse Source

定时任务执行配置

master
CMM 2 months ago
parent
58a4c39de3
commit
b1d0c9afb9
6 changed files with 150 additions and 124 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +49
    -43
      kq-vas-api/src/main/java/com/ningdatech/kqapi/scheduler/task/CheckMattersUrlTask.java
  2. +58
    -50
      kq-vas-api/src/main/java/com/ningdatech/kqapi/scheduler/task/ItemSynTask.java
  3. +1
    -4
      kq-vas-api/src/main/java/com/ningdatech/kqapi/scheduler/task/ItemSynTestTask.java
  4. +34
    -27
      kq-vas-api/src/main/java/com/ningdatech/kqapi/scheduler/task/SynTask.java
  5. +4
    -0
      kq-vas-api/src/main/resources/application-dev.yml
  6. +4
    -0
      kq-vas-api/src/main/resources/application-prod.yml

+ 49
- 43
kq-vas-api/src/main/java/com/ningdatech/kqapi/scheduler/task/CheckMattersUrlTask.java View File

@@ -47,6 +47,9 @@ public class CheckMattersUrlTask {
@Value("${hostname}")
public String HOST;

@Value("${task.switch.is-open}")
private boolean flag;

@Value("${spring.profiles.active}")
public String active;

@@ -64,52 +67,55 @@ public class CheckMattersUrlTask {
// return;
// }

StopWatch stopWatch = new StopWatch();
stopWatch.start();

log.info("==========开始校检事项表里的链接 会不会404 如果404就隐藏");

List<KqZzsfwMenu> menus = menuService.list(Wrappers.lambdaQuery(KqZzsfwMenu.class)
.eq(KqZzsfwMenu::getHasUrl, 1)
.isNotNull(KqZzsfwMenu::getWebapplyurl));
Integer errNum = 0;
List<String> errUrls = Lists.newArrayList();

for (KqZzsfwMenu menu : menus) {
RestTemplate restTemplate = new RestTemplate(generateHttpRequestFactory());
String url = "https://www.zjzwfw.gov.cn/jpaas-zjservice-server/open-api/item/getItemDetail";
MultiValueMap<String, String> map = new LinkedMultiValueMap<>();

String webapplyurl = menu.getWebapplyurl();
webapplyurl = webapplyurl.replace("/#","");
Map<String, String> queryParams = getQueryParams(new URL(webapplyurl));
String localInnerCode = queryParams.get("localInnerCode");
log.info("localInnerCode :{}",localInnerCode);
map.add("localInnerCode", localInnerCode);
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED);
HttpEntity<MultiValueMap<String, String>> formEntity = new HttpEntity(map, headers);
ResponseEntity<Item> response = restTemplate.postForEntity(url, formEntity, Item.class);
System.out.println(response.getBody());
Item body = response.getBody();
assert body != null;
Item.ItemData itemData = body.getData();
//如果是空的
if(Objects.isNull(itemData.getItem())){
//隐藏
menuService.update(Wrappers.lambdaUpdate(KqZzsfwMenu.class)
.eq(KqZzsfwMenu::getId,menu.getId())
.set(KqZzsfwMenu::getHasUrl,0)
.set(KqZzsfwMenu::getWebapplyurl,null));
errNum += 1;
errUrls.add(webapplyurl);
// 线上环境运行
if (flag && HOST.equals(InetAddress.getLocalHost().getHostName())) {
StopWatch stopWatch = new StopWatch();
stopWatch.start();

log.info("==========开始校检事项表里的链接 会不会404 如果404就隐藏");

List<KqZzsfwMenu> menus = menuService.list(Wrappers.lambdaQuery(KqZzsfwMenu.class)
.eq(KqZzsfwMenu::getHasUrl, 1)
.isNotNull(KqZzsfwMenu::getWebapplyurl));
Integer errNum = 0;
List<String> errUrls = Lists.newArrayList();

for (KqZzsfwMenu menu : menus) {
RestTemplate restTemplate = new RestTemplate(generateHttpRequestFactory());
String url = "https://www.zjzwfw.gov.cn/jpaas-zjservice-server/open-api/item/getItemDetail";
MultiValueMap<String, String> map = new LinkedMultiValueMap<>();

String webapplyurl = menu.getWebapplyurl();
webapplyurl = webapplyurl.replace("/#", "");
Map<String, String> queryParams = getQueryParams(new URL(webapplyurl));
String localInnerCode = queryParams.get("localInnerCode");
log.info("localInnerCode :{}", localInnerCode);
map.add("localInnerCode", localInnerCode);
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED);
HttpEntity<MultiValueMap<String, String>> formEntity = new HttpEntity(map, headers);
ResponseEntity<Item> response = restTemplate.postForEntity(url, formEntity, Item.class);
System.out.println(response.getBody());
Item body = response.getBody();
assert body != null;
Item.ItemData itemData = body.getData();
//如果是空的
if (Objects.isNull(itemData.getItem())) {
//隐藏
menuService.update(Wrappers.lambdaUpdate(KqZzsfwMenu.class)
.eq(KqZzsfwMenu::getId, menu.getId())
.set(KqZzsfwMenu::getHasUrl, 0)
.set(KqZzsfwMenu::getWebapplyurl, null));
errNum += 1;
errUrls.add(webapplyurl);
}
}
}

log.info("不可用的链接有 :{}条 ,详情:{}",errNum,errUrls);
log.info("不可用的链接有 :{}条 ,详情:{}", errNum, errUrls);

stopWatch.stop();
log.info("校检事项表里的链接结束====={}s",stopWatch.getTotalTimeSeconds());
stopWatch.stop();
log.info("校检事项表里的链接结束====={}s", stopWatch.getTotalTimeSeconds());
}
}

public static void main(String[] args) throws Exception {


+ 58
- 50
kq-vas-api/src/main/java/com/ningdatech/kqapi/scheduler/task/ItemSynTask.java View File

@@ -1,5 +1,7 @@
package com.ningdatech.kqapi.scheduler.task;

import java.net.InetAddress;
import java.net.UnknownHostException;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.util.List;
@@ -41,6 +43,9 @@ public class ItemSynTask {
@Value("${hostname}")
public String HOST;

@Value("${task.switch.is-open}")
private boolean flag;

private final IDscSxAdsShareItemQltQlsxCommonIDVKqService kqService;
private final IrsManage irsManage;
private final CachePlusOps cachePlusOps;
@@ -51,63 +56,66 @@ public class ItemSynTask {
* 每天凌晨5点同步数据
*/
@Scheduled(cron = "0 0 5 * * ?")
public void synData() {
log.info("事项数据 ===================== 同步开始");
StopWatch stopWatch = new StopWatch();
stopWatch.start();
// 每天凌晨5点循环拉取较前一天新更新的数据
// 根据事项表中最后更新日期 循环更新至前一天的事项数据
// 默认一页返回
int pageNum = 1;
// irs接口最大返回数据量1000条
int pageSize = 1000;
public void synData() throws UnknownHostException {
// 线上环境运行
if (flag && HOST.equals(InetAddress.getLocalHost().getHostName())) {
log.info("事项数据 ===================== 同步开始");
StopWatch stopWatch = new StopWatch();
stopWatch.start();
// 每天凌晨5点循环拉取较前一天新更新的数据
// 根据事项表中最后更新日期 循环更新至前一天的事项数据
// 默认一页返回
int pageNum = 1;
// irs接口最大返回数据量1000条
int pageSize = 1000;

List<DscSxAdsShareItemQltQlsxCommonIDVKq> records = Lists.newArrayList();
List<DscSxAdsShareItemQltQlsxCommonIDVKq> records = Lists.newArrayList();

// 从缓存中取上次同步数据中最新的更新日期
CacheKey key = new CacheKey(ITEM_BASIC_INFO_KEY);
String start = cachePlusOps.get(key);
// 如果缓存中不存在 从表中查询
if (StringUtils.isBlank(start)){
start = kqService.getNewestUpdateDate();
}
String end = NdDateUtils.format(LocalDateTime.now().minusDays(1), NdDateUtils.DEFAULT_DATE_FORMAT);
LocalDate startDate = LocalDate.parse(start).plusDays(1);
LocalDate endDate = LocalDate.parse(end);
for (LocalDate date = startDate; !date.isAfter(endDate); date = date.plusDays(1)) {
boolean flag = true;
while (flag) {
String startDateStr = NdDateUtils.format(date, NdDateUtils.DEFAULT_DATE_FORMAT);
String endDateStr = NdDateUtils.format(date.plusDays(1), NdDateUtils.DEFAULT_DATE_FORMAT);
Object listObject = irsManage.itemBasicInfoList(startDateStr, endDateStr, pageSize, pageNum);
// 如果数据正常返回
if (listObject instanceof JSONArray) {
JSONArray listJsonArray = (JSONArray) listObject;
log.info("事项列表数量为：{}", listJsonArray.size());
getSaveRecords(listJsonArray, records);
// 如果listJsonArray数据量等于1000 需要继续拉取数据
if (listJsonArray.size() == pageSize) {
pageNum++;
// 从缓存中取上次同步数据中最新的更新日期
CacheKey key = new CacheKey(ITEM_BASIC_INFO_KEY);
String start = cachePlusOps.get(key);
// 如果缓存中不存在 从表中查询
if (StringUtils.isBlank(start)) {
start = kqService.getNewestUpdateDate();
}
String end = NdDateUtils.format(LocalDateTime.now().minusDays(1), NdDateUtils.DEFAULT_DATE_FORMAT);
LocalDate startDate = LocalDate.parse(start).plusDays(1);
LocalDate endDate = LocalDate.parse(end);
for (LocalDate date = startDate; !date.isAfter(endDate); date = date.plusDays(1)) {
boolean flag = true;
while (flag) {
String startDateStr = NdDateUtils.format(date, NdDateUtils.DEFAULT_DATE_FORMAT);
String endDateStr = NdDateUtils.format(date.plusDays(1), NdDateUtils.DEFAULT_DATE_FORMAT);
Object listObject = irsManage.itemBasicInfoList(startDateStr, endDateStr, pageSize, pageNum);
// 如果数据正常返回
if (listObject instanceof JSONArray) {
JSONArray listJsonArray = (JSONArray) listObject;
log.info("事项列表数量为：{}", listJsonArray.size());
getSaveRecords(listJsonArray, records);
// 如果listJsonArray数据量等于1000 需要继续拉取数据
if (listJsonArray.size() == pageSize) {
pageNum++;
} else {
flag = false;
}
} else {
flag = false;
log.error("事项列表数据异常：{}", listObject.toString());
return;
}
} else {
log.error("事项列表数据异常：{}", listObject.toString());
return;
}
}
log.info("{} 数据列表数量为：{}", startDate, records.size());
kqService.saveBatch(records);
log.info("事项同步成功，更新时间：{}", startDate);
// 更新缓存中的更新时间
// 获取此次保存记录中的最新更新时间
records.sort((o1, o2) -> o2.getUpdateDate().compareTo(o1.getUpdateDate()));
LocalDateTime updateDate = records.get(0).getUpdateDate();
String updateDateStr = NdDateUtils.format(updateDate, NdDateUtils.DEFAULT_DATE_FORMAT);
cachePlusOps.set(key, updateDateStr);
stopWatch.stop();
log.info("事项数据 ===================== 同步总共耗时 :{} s", stopWatch.getTotalTimeSeconds());
}
log.info("{} 数据列表数量为：{}", startDate, records.size());
kqService.saveBatch(records);
log.info("事项同步成功，更新时间：{}", startDate);
// 更新缓存中的更新时间
// 获取此次保存记录中的最新更新时间
records.sort((o1, o2) -> o2.getUpdateDate().compareTo(o1.getUpdateDate()));
LocalDateTime updateDate = records.get(0).getUpdateDate();
String updateDateStr = NdDateUtils.format(updateDate, NdDateUtils.DEFAULT_DATE_FORMAT);
cachePlusOps.set(key, updateDateStr);
stopWatch.stop();
log.info("事项数据 ===================== 同步总共耗时 :{} s", stopWatch.getTotalTimeSeconds());
}

private void getSaveRecords(JSONArray listJsonArray, List<DscSxAdsShareItemQltQlsxCommonIDVKq> records) {


+ 1
- 4
kq-vas-api/src/main/java/com/ningdatech/kqapi/scheduler/task/ItemSynTestTask.java View File

@@ -50,10 +50,7 @@ public class ItemSynTestTask {

String transmitListUrl = "https://qyxspt.kq.gov.cn:33060/kq/api/v1/irs/item-basic-info-list";
String transmitDetailUrl = "https://qyxspt.kq.gov.cn:33060/kq/api/v1/irs/item-basic-info-detail";
/**
* 每天凌晨5点同步数据
*/
//@Scheduled(cron = "0 0 5 * * ?")

public void synData() {
log.info("事项数据 ===================== 同步开始");
StopWatch stopWatch = new StopWatch();


+ 34
- 27
kq-vas-api/src/main/java/com/ningdatech/kqapi/scheduler/task/SynTask.java View File

@@ -21,6 +21,7 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;

import java.net.InetAddress;
import java.net.UnknownHostException;
import java.time.LocalDateTime;
import java.util.ArrayList;
@@ -40,6 +41,9 @@ public class SynTask {
@Value("${hostname}")
public String HOST;

@Value("${task.switch.is-open}")
private boolean flag;

@Value("${spring.profiles.active}")
public String active;

@@ -70,36 +74,39 @@ public class SynTask {
*/
@Scheduled(cron = "0 30 8 * * ?")
public void synData() throws UnknownHostException {
log.info("政务数据 ===================== 同步开始");
StopWatch stopWatch = new StopWatch();
stopWatch.start();

//1.处理 去重数据
if (matterCacheHelper.refreshAll()) {
matterManage.removeAllDup();
List<KqZzsfwMattersDeduplicate> allDup = matterCacheHelper.all();
matterDeduplicateService.saveBatch(allDup);

//2. 事项
List<KqZzsfwMenu> menus = menuService.list();
for (KqZzsfwMenu menu : menus) {
String url = matterCacheHelper.getUrl(menu.getItemName());
if (StringUtils.isBlank(url)) {
menuService.update(Wrappers.lambdaUpdate(KqZzsfwMenu.class)
.eq(KqZzsfwMenu::getId, menu.getId())
.set(KqZzsfwMenu::getWebapplyurl, null)
.set(KqZzsfwMenu::getHasUrl, CommonEnum.NO.getCode()));
} else {
menuService.update(Wrappers.lambdaUpdate(KqZzsfwMenu.class)
.eq(KqZzsfwMenu::getId, menu.getId())
.set(KqZzsfwMenu::getWebapplyurl, url)
.set(KqZzsfwMenu::getHasUrl, CommonEnum.YES.getCode()));
// 线上环境运行
if (flag && HOST.equals(InetAddress.getLocalHost().getHostName())) {
log.info("政务数据 ===================== 同步开始");
StopWatch stopWatch = new StopWatch();
stopWatch.start();

//1.处理 去重数据
if (matterCacheHelper.refreshAll()) {
matterManage.removeAllDup();
List<KqZzsfwMattersDeduplicate> allDup = matterCacheHelper.all();
matterDeduplicateService.saveBatch(allDup);

//2. 事项
List<KqZzsfwMenu> menus = menuService.list();
for (KqZzsfwMenu menu : menus) {
String url = matterCacheHelper.getUrl(menu.getItemName());
if (StringUtils.isBlank(url)) {
menuService.update(Wrappers.lambdaUpdate(KqZzsfwMenu.class)
.eq(KqZzsfwMenu::getId, menu.getId())
.set(KqZzsfwMenu::getWebapplyurl, null)
.set(KqZzsfwMenu::getHasUrl, CommonEnum.NO.getCode()));
} else {
menuService.update(Wrappers.lambdaUpdate(KqZzsfwMenu.class)
.eq(KqZzsfwMenu::getId, menu.getId())
.set(KqZzsfwMenu::getWebapplyurl, url)
.set(KqZzsfwMenu::getHasUrl, CommonEnum.YES.getCode()));
}
}
}
}

stopWatch.stop();
log.info("政务数据 ===================== 同步总共耗时 :{} s", stopWatch.getTotalTimeSeconds());
stopWatch.stop();
log.info("政务数据 ===================== 同步总共耗时 :{} s", stopWatch.getTotalTimeSeconds());
}
}

public void synMenu() {


+ 4
- 0
kq-vas-api/src/main/resources/application-dev.yml View File

@@ -130,7 +130,11 @@ swagger:
sys:
web-url: http://kqzh.ningdatech.com

# 定时任务配置
hostname: iZbp13nwyvib53j4j1p2xoZ
task:
switch:
is-open: false #开关
jasypt:
encryptor:
password: CodeSheep


+ 4
- 0
kq-vas-api/src/main/resources/application-prod.yml View File

@@ -124,7 +124,11 @@ log:
swagger:
enabled: false

# 定时任务配置
hostname: iZr3i038ar3e2l7qn9ji2oZ
task:
switch:
is-open: true #开关

sys:
web-url: https://qyxspt.kq.gov.cn:33060


Write Preview
Loading…
Cancel
Save