数据同步

kqapi/src/main/java/com/ningdatech/kqapi/common/helper/MatterCacheHelper.java

@@ -0,0 +1,26 @@
+package com.ningdatech.kqapi.common.helper;
+
+
+import com.ningdatech.kqapi.zzsfw.entity.entity.DscSxAdsShareItemQltQlsxCommonIDVKq;
+import com.ningdatech.kqapi.zzsfw.entity.entity.NdKqZzsfwMattersDeduplicate;
+
+import java.util.List;
+
+/**
+ * <p>
+ * MatterCacheHelper -
+ * </p>
+ *
+ * @author ZPF
+ * @since 13:50 2024/1/3
+ */
+public interface MatterCacheHelper {
+
+    NdKqZzsfwMattersDeduplicate getMatter(String qlName,String webapplyurl);
+
+    String getUrl(String qlName);
+
+    List<NdKqZzsfwMattersDeduplicate> all();
+
+    Boolean refreshAll();
+}

kqapi/src/main/java/com/ningdatech/kqapi/common/helper/basic/AbstractMatterCacheHelper.java

@@ -0,0 +1,120 @@
+package com.ningdatech.kqapi.common.helper.basic;
+
+import com.baomidou.mybatisplus.core.toolkit.Wrappers;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.ningdatech.kqapi.common.constant.BizConst;
+import com.ningdatech.kqapi.zzsfw.entity.entity.DscSxAdsShareItemQltQlsxCommonIDVKq;
+import com.ningdatech.kqapi.zzsfw.entity.entity.MatterKey;
+import com.ningdatech.kqapi.zzsfw.entity.entity.NdKqZzsfwMattersDeduplicate;
+import com.ningdatech.kqapi.zzsfw.service.IDscSxAdsShareItemQltQlsxCommonIDVKqService;
+import com.ningdatech.kqapi.zzsfw.service.INdKqZzsfwMatterDeduplicateService;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+/**
+ * <p>
+ * AbstractMatterCache
+ * </p>
+ *
+ * @author ZPF
+ * @since 14:41 2023/3/1
+ */
+@Slf4j
+public abstract class AbstractMatterCacheHelper implements InitializingBean {
+
+    @Autowired
+    private IDscSxAdsShareItemQltQlsxCommonIDVKqService kqService;
+
+    @Autowired
+    private INdKqZzsfwMatterDeduplicateService deduplicateService;
+
+    protected LoadingCache<MatterKey, NdKqZzsfwMattersDeduplicate> mattersDupCache;
+
+    protected LoadingCache<String, String> mattersUrlCache;
+
+    private Boolean initMatterCache() {
+        List<DscSxAdsShareItemQltQlsxCommonIDVKq> allMatters = kqService
+                .list(Wrappers.lambdaQuery(DscSxAdsShareItemQltQlsxCommonIDVKq.class)
+                .select(DscSxAdsShareItemQltQlsxCommonIDVKq::getRowguid,DscSxAdsShareItemQltQlsxCommonIDVKq::getQlName,
+                        DscSxAdsShareItemQltQlsxCommonIDVKq::getWebapplyurl)
+                .isNotNull(DscSxAdsShareItemQltQlsxCommonIDVKq::getWebapplyurl));
+        if (allMatters.isEmpty()) {
+            log.warn("事项数据未初始化");
+            return Boolean.FALSE;
+        }
+        allMatters.forEach(m -> {
+            MatterKey key = MatterKey.of(m.getQlName(),m.getWebapplyurl());
+            mattersUrlCache.put(m.getQlName(),m.getWebapplyurl());
+            if(mattersDupCache.asMap().containsKey(key)){
+                NdKqZzsfwMattersDeduplicate dup = mattersDupCache.get(key);
+                dup.setCountNum(dup.getCountNum() + 1);
+                mattersDupCache.put(key,dup);
+            }else{
+                mattersDupCache.put(key,NdKqZzsfwMattersDeduplicate.of(m.getQlName(),m.getWebapplyurl()));
+            }
+        });
+        return Boolean.TRUE;
+    }
+
+    protected NdKqZzsfwMattersDeduplicate get(String qlName,String url) {
+        MatterKey key = MatterKey.of(qlName, url);
+        return mattersDupCache.get(key);
+    }
+
+    protected String getUrl(String qlName) {
+        return mattersUrlCache.get(qlName);
+    }
+
+    protected List<NdKqZzsfwMattersDeduplicate> all() {
+        return Lists.newArrayList(mattersDupCache.asMap().values());
+    }
+
+    @Override
+    public void afterPropertiesSet() {
+        mattersUrlCache = Caffeine.newBuilder()
+                .refreshAfterWrite(7, TimeUnit.DAYS)
+                .maximumSize(1024 * 100)
+                .build(key -> {
+                    DscSxAdsShareItemQltQlsxCommonIDVKq matter = kqService.getOne(Wrappers.lambdaQuery(DscSxAdsShareItemQltQlsxCommonIDVKq.class)
+                            .eq(DscSxAdsShareItemQltQlsxCommonIDVKq::getQlName, key)
+                            .last(BizConst.LIMIT_1)
+                            .isNotNull(DscSxAdsShareItemQltQlsxCommonIDVKq::getWebapplyurl));
+                    if (Objects.isNull(matter)) {
+                        //查不到直接返回null
+                        return null;
+                    }
+                    return matter.getWebapplyurl();
+                });
+        mattersDupCache = Caffeine.newBuilder()
+                .refreshAfterWrite(7, TimeUnit.DAYS)
+                .maximumSize(1024 * 100)
+                .build(key -> {
+                    NdKqZzsfwMattersDeduplicate dup = deduplicateService.getOne(Wrappers.lambdaQuery(NdKqZzsfwMattersDeduplicate.class)
+                            .eq(NdKqZzsfwMattersDeduplicate::getQlName, key)
+                            .last(BizConst.LIMIT_1)
+                            .isNotNull(NdKqZzsfwMattersDeduplicate::getWebapplyurl));
+                    if (Objects.isNull(dup)) {
+                        //查不到直接返回null
+                        return null;
+                    }
+                    return dup;
+                });
+        // 初始化所有事项数据到缓存
+        initMatterCache();
+    }
+
+    public Boolean init() {
+        // 初始化所有事项数据到缓存
+        return initMatterCache();
+    }
+}

kqapi/src/main/java/com/ningdatech/kqapi/common/helper/impl/MattersCacheHelperImpl.java

@@ -0,0 +1,46 @@
+package com.ningdatech.kqapi.common.helper.impl;
+
+import com.ningdatech.kqapi.common.helper.MatterCacheHelper;
+import com.ningdatech.kqapi.common.helper.basic.AbstractMatterCacheHelper;
+import com.ningdatech.kqapi.zzsfw.entity.entity.NdKqZzsfwMattersDeduplicate;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+
+/**
+ * @author ZPF
+ * @date 2024/1/3 上午8:58
+ */
+@Slf4j
+@Component
+public class MattersCacheHelperImpl extends AbstractMatterCacheHelper implements MatterCacheHelper {
+
+
+    @Override
+    public NdKqZzsfwMattersDeduplicate getMatter(String qlName,String webapplyurl) {
+        if (StringUtils.isBlank(qlName) || StringUtils.isBlank(webapplyurl)) {
+            return null;
+        }
+        return super.get(qlName, webapplyurl);
+    }
+
+    @Override
+    public String getUrl(String qlName) {
+        if (StringUtils.isBlank(qlName)) {
+            return null;
+        }
+        return super.getUrl(qlName);
+    }
+
+    @Override
+    public List<NdKqZzsfwMattersDeduplicate> all() {
+        return super.all();
+    }
+
+    @Override
+    public Boolean refreshAll() {
+        return super.init();
+    }
+}

kqapi/src/main/java/com/ningdatech/kqapi/zzsfw/controller/NdKqZzsfwMenuController.java

@@ -69,4 +69,10 @@ public class NdKqZzsfwMenuController {
     public String removeAllDup() {
         return matterManage.removeAllDup();
     }
+
+    @ApiOperation(value = "同步数据", notes = "同步数据")
+    @GetMapping("/syn")
+    public String synData() {
+        return matterManage.synData();
+    }
 }

kqapi/src/main/java/com/ningdatech/kqapi/zzsfw/entity/entity/MatterKey.java

@@ -0,0 +1,24 @@
+package com.ningdatech.kqapi.zzsfw.entity.entity;
+
+import lombok.Data;
+
+/**
+ * @Classname MatterKey
+ * @Description
+ * @Date 2024/1/3 11:13
+ * @Author PoffyZhang
+ */
+@Data
+public class MatterKey {
+
+    private String qlName;
+
+    private String webapplyurl;
+
+    public static MatterKey of(String qlName, String webapplyurl) {
+        MatterKey matterKey = new MatterKey();
+        matterKey.setQlName(qlName);
+        matterKey.setWebapplyurl(webapplyurl);
+        return matterKey;
+    }
+}

kqapi/src/main/java/com/ningdatech/kqapi/zzsfw/entity/entity/NdKqZzsfwMattersDeduplicate.java

@@ -39,5 +39,13 @@ public class NdKqZzsfwMattersDeduplicate implements Serializable {
     private String webapplyurl;
 
     @ApiModelProperty("重复数")
-    private Integer countNum;
+    private Integer countNum = 0;
+
+    public static NdKqZzsfwMattersDeduplicate of(String qlName, String webapplyurl) {
+        NdKqZzsfwMattersDeduplicate dup = new NdKqZzsfwMattersDeduplicate();
+        dup.setQlName(qlName);
+        dup.setWebapplyurl(webapplyurl);
+        dup.setCountNum(1);
+        return dup;
+    }
 }

kqapi/src/main/java/com/ningdatech/kqapi/zzsfw/manage/MatterManage.java

@@ -2,14 +2,15 @@ package com.ningdatech.kqapi.zzsfw.manage;
 
 import cn.hutool.core.bean.BeanUtil;
 import cn.hutool.core.collection.CollUtil;
+import cn.hutool.core.date.StopWatch;
 import com.baomidou.mybatisplus.core.toolkit.Wrappers;
 import com.ningdatech.kqapi.common.enumeration.CommonEnum;
+import com.ningdatech.kqapi.common.helper.MatterCacheHelper;
 import com.ningdatech.kqapi.zzsfw.constants.ZzsfwMenuConstant;
 import com.ningdatech.kqapi.zzsfw.entity.dto.NdKqZzsfwMattersDeduplicateDTO;
 import com.ningdatech.kqapi.zzsfw.entity.dto.NdKqZzsfwMenuDTO;
 import com.ningdatech.kqapi.zzsfw.entity.entity.NdKqZzsfwMattersDeduplicate;
 import com.ningdatech.kqapi.zzsfw.entity.entity.NdKqZzsfwMenu;
-import com.ningdatech.kqapi.zzsfw.entity.entity.NdKqZzsfwPolicy;
 import com.ningdatech.kqapi.zzsfw.entity.vo.MatterTopVO;
 import com.ningdatech.kqapi.zzsfw.entity.vo.TreeVO;
 import com.ningdatech.kqapi.zzsfw.service.INdKqZzsfwMatterDeduplicateService;
@@ -37,6 +38,8 @@ public class MatterManage {
 
     private final INdKqZzsfwMatterDeduplicateService matterDeduplicateService;
 
+    private final MatterCacheHelper matterCacheHelper;
+
     public List<TreeVO> getMatters() {
         List<NdKqZzsfwMenu> matters = menuService.list(Wrappers.lambdaQuery(NdKqZzsfwMenu.class)
                 .isNotNull(NdKqZzsfwMenu::getWebapplyurl)
@@ -146,4 +149,38 @@ public class MatterManage {
         matterDeduplicateService.remove(Wrappers.lambdaQuery(NdKqZzsfwMattersDeduplicate.class));
         return "删除成功";
     }
+
+    public String synData() {
+        log.info("政务数据 ===================== 同步开始");
+        StopWatch stopWatch = new StopWatch();
+        stopWatch.start();
+
+        //1.处理 去重数据
+        if(matterCacheHelper.refreshAll()){
+            removeAllDup();
+            List<NdKqZzsfwMattersDeduplicate> allDup = matterCacheHelper.all();
+            matterDeduplicateService.saveBatch(allDup);
+
+            //2. 事项
+            List<NdKqZzsfwMenu> menus = menuService.list();
+            for(NdKqZzsfwMenu menu : menus){
+                String url = matterCacheHelper.getUrl(menu.getItemName());
+                if(StringUtils.isBlank(url)){
+                    menuService.update(Wrappers.lambdaUpdate(NdKqZzsfwMenu.class)
+                            .eq(NdKqZzsfwMenu::getId,menu.getId())
+                            .set(NdKqZzsfwMenu::getWebapplyurl,null)
+                            .set(NdKqZzsfwMenu::getHasUrl,CommonEnum.NO.getCode()));
+                }else{
+                    menuService.update(Wrappers.lambdaUpdate(NdKqZzsfwMenu.class)
+                            .eq(NdKqZzsfwMenu::getId,menu.getId())
+                            .set(NdKqZzsfwMenu::getWebapplyurl,url)
+                            .set(NdKqZzsfwMenu::getHasUrl,CommonEnum.YES.getCode()));
+                }
+            }
+        }
+
+        stopWatch.stop();
+        log.info("政务数据 ===================== 同步总共耗时 :{} s",stopWatch.getTotalTimeSeconds());
+        return "同步成功 耗时 :" + stopWatch.getTotalTimeSeconds() + "s";
+    }
 }