dup_check/main1.py

# coding=utf-8
import sys
import re
import mysql_pool
from pymysql.converters import escape_string
import cosin_similarity
import pandas as pd
import datetime
import requests
import os
import pymysql

wdys1 = {
    "项目名称": "xmmc",
    "现状问题": "xzwt",
    "系统基础": "xtjc",
    "项目目标": "xmmb",
    "预期绩效": "yqjx",
    "建设需求": "jsxq",
    "数据需求": "sjxq",
    "安全需求": "aqxq",
    "业务领域": "ywly",
    "核心业务": "hxyw",
    "业务需求": "ywxq",
    "业务协同": "ywxt",
    "建设层级": "jscj",
    "用户范围": "yhfw",
    "目标群体": "mbqt",
    "建设内容": "jsnr",
    "功能模块": "gnmk",
    "数据共享": "sjgx",
    "智能要素": "znys"
}
wdys2 = {
    "xmmc": "项目名称",
    "xzwt": "现状问题",
    "xtjc": "系统基础",
    "xmmb": "项目目标",
    "yqjx": "预期绩效",
    "jsxq": "建设需求",
    "sjxq": "数据需求",
    "aqxq": "安全需求",
    "ywly": "业务领域",
    "hxyw": "核心业务",
    "ywxq": "业务需求",
    "ywxt": "业务协同",
    "jscj": "建设层级",
    "yhfw": "用户范围",
    "mbqt": "目标群体",
    "jsnr": "建设内容",
    "gnmk": "功能模块",
    "sjgx": "数据共享",
    "znys": "智能要素"
}
gnmkys = {
    "gnmc": "功能名称",
    "gnms": "功能描述"
}


def getFlag():
    data_dict = {}
    df = pd.read_excel("0825-丽水系统查重维度.xlsx")
    data = df.values
    data = list(pd.Series(data[:, 1]).dropna())
    for d in data:
        try:
            wd = re.search("(.*?)（.*?%）", d).group(1).strip()
            wdc = wdys1.get(wd)
            if wdc:
                qz = re.search(".*?（(.*?%)）", d).group(1)
                data_dict[wdc] = qz
        except:
            pass
    return data_dict


def gong_neng_mo_kuai(mysql, dl, data, er_title):
    # 将excel文件中的所有第三维度内容进行拼接
    str_dict = {}
    for et in er_title:
        for d in data:
            if d[1] == et:
                if str_dict.get(et):
                    str_dict[et] = str_dict.get(et) + d[3]
                else:
                    str_dict[et] = d[3]

    for k, v in str_dict.items():
        mysql.sql_change_msg(
            """insert into idc_project_module (project_id, check_duplicate_count, module_name, module_content, create_time, update_time, tag) value(%d, dup_file_test, "%s", "%s", "%s", "%s", "模块")""" % (
                int(dl[0]), k, v, str(datetime.datetime.now())[:-7], str(datetime.datetime.now())[:-7]))

    module_id_list = mysql.sql_select_many(
        """select project_module_id, module_name, module_content  from idc_project_module where project_id=%d""" % dl[
            0])
    data_list = []
    for mil in module_id_list:
        data_dict = {}
        data_dict["project_module_id"] = mil.get("project_module_id")
        data_dict["gnmc"] = mil.get("module_name")
        data_dict["gnms"] = mil.get("module_content")
        data_list.append(data_dict)
    # print(data_list)
    for i in data_list:
        gnmk_copy1 = mysql.sql_select_many("""select * from user_history_module_data""")
        if gnmk_copy1:
            for gc in gnmk_copy1:
                total_similarity1 = 0
                total_keywords1 = []
                total_similarity2 = 0
                total_keywords2 = []
                mysql.sql_change_msg(
                    """insert into idc_project_module_check (project_module_id, module_name, project_name, company_name, create_time, update_time) value(%d, "%s", "%s", "%s", "%s", "%s")"""
                    % (
                        i.get("project_module_id"), gc.get("gnmc"), gc.get("xmmc"), "",
                        str(datetime.datetime.now())[:-7],
                        str(datetime.datetime.now())[:-7]))
                dup_module_id = mysql.cur.lastrowid
                for j in ["gnmc", "gnms"]:
                    # 循环遍历每一个模块名称
                    content_x = gc.get(j)
                    content_y = i.get(j)
                    if content_x and content_y:
                        if j == "gnmc":
                            similarity = cosin_similarity.CosineSimilarity(content_x, content_y)
                            # 相似度  关键词
                            similarity, keyword_x, keywords = similarity.main()
                            similarity = similarity * 1
                            total_keywords1 += keywords
                            #print("######################相似度: %.2f%%" % similarity, "关键词: %s" % keywords)
                            # 相似度相加
                            total_similarity1 += similarity
                            mysql.sql_change_msg(
                                """insert into idc_project_module_check_detail (dup_module_id, project_name, module_content, dup_module_content, similarity, dimension, create_time, update_time) value (%d, "%s", "%s", "%s", %f, "%s", "%s", "%s")"""
                                % (dup_module_id, dl[2], escape_string(content_y), escape_string(content_x), similarity,
                                   "功能名称",
                                   str(datetime.datetime.now())[:-7], str(datetime.datetime.now())[:-7]))
                        else:
                            similarity = cosin_similarity.CosineSimilarity(content_x, content_y)
                            # 相似度  关键词
                            similarity, keyword_x, keywords = similarity.main()
                            similarity = similarity * 99
                            total_keywords2 += keywords
                            #print("######################相似度: %.2f%%" % similarity, "关键词: %s" % keywords)
                            # 相似度相加
                            total_similarity2 += similarity
                            mysql.sql_change_msg(
                                """insert into idc_project_module_check_detail (dup_module_id, project_name, module_content, dup_module_content, similarity, dimension, create_time, update_time) value (%d, "%s", "%s", "%s", %f, "%s", "%s", "%s")"""
                                % (dup_module_id, dl[2], escape_string(content_y), escape_string(content_x), similarity,
                                   "功能模块描述",
                                   str(datetime.datetime.now())[:-7], str(datetime.datetime.now())[:-7]))
                mysql.sql_change_msg("""update idc_project_module_check set similarity=%f where dup_module_id=%d""" % (
                    total_similarity1 + total_similarity2, dup_module_id))
            gnmk_gjc = {}
            for a in ["gnmc", "gnms"]:
                if i.get(a):
                    content_x = i.get(a)
                    content_y = i.get(a)
                    if a == "gnmc":
                        similarity = cosin_similarity.CosineSimilarity(content_x, content_y)
                        # 相似度  关键词
                        similarity, keyword_x, keywords = similarity.main()
                        gnmk_gjc[a] = keywords
                    else:
                        similarity = cosin_similarity.CosineSimilarity(content_x, content_y)
                        # 相似度  关键词
                        similarity, keyword_x, keywords = similarity.main()
                        gnmk_gjc[a] = keywords
            mysql.sql_change_msg("""insert into user_module_keywords (xmmc, gnmc, gnms) value("%s", "%s", "%s")""" % (
                dl[2], str(gnmk_gjc.get("gnmc"))[1:-1] if gnmk_gjc.get("gnmc") else None,
                str(gnmk_gjc.get("gnms"))[1:-1] if gnmk_gjc.get("gnms") else None))


def project_check(data_list):
    mysql = mysql_pool.ConnMysql()
    # mysql.sql_select_many("""select * from mkgjc""")
    # 读取维度和权重
    # xmnr_count = len(mysql.sql_select_many("""select * from xmnr_copy1"""))
    # gnmk_count = len(mysql.sql_select_many("""select * from gnmk_copy1"""))
    xmnr_count = len(mysql.sql_select_many("""select * from user_history_data"""))
    gnmk_count = len(mysql.sql_select_many("""select * from user_history_module_data"""))
    get_data_dict = getFlag()
    # 遍历excel存储路径
    for dl in data_list:
        # path = "0825-丽水系统查重维度1.xlsx"
        # 读取路径下的excel
        print(dl,dl[1])
        df = pd.read_excel(dl[1])
        data = df.values
        # 将excel文件中的所有维度内容进行拼接
        join_str = ""
        str_dict = {}
        title = ""
        er_title = set()
        for d in data:
            if pd.notnull(d[0]):
                title = d[0]
                if title == "功能模块":
                    er_title.add(d[1])
                join_str = ""
                for i in d[1:]:
                    if pd.notnull(i):
                        join_str += i
                str_dict[wdys1.get(title)] = join_str
            else:
                if title == "功能模块":
                    er_title.add(d[1])
                for i in d[1:]:
                    if pd.notnull(i):
                        join_str += i
                str_dict[wdys1.get(title)] = str_dict.get(wdys1.get(title)) + join_str
        print(str_dict)
        mysql.sql_change_msg(
            """insert into user_data (xmmc, xzwt, xtjc, xmmb, yqjx, jsxq, sjxq, aqxq, ywly, hxyw, ywxq, ywxt, jscj, yhfw, mbqt, jsnr, gnmk, sjgx, znys) value ("%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s")"""
            % (dl[0], str_dict.get("xzwt") if str_dict.get("xzwt") else None,
               str_dict.get("xtjc") if str_dict.get("xtjc") else None,
               str_dict.get("xmmb") if str_dict.get("xmmb") else None,
               str_dict.get("yqjx") if str_dict.get("yqjx") else None,
               str_dict.get("jsxq") if str_dict.get("jsxq") else None,
               str_dict.get("sjxq") if str_dict.get("sjxq") else None,
               str_dict.get("aqxq") if str_dict.get("aqxq") else None,
               str_dict.get("ywly") if str_dict.get("ywly") else None,
               str_dict.get("hxyw") if str_dict.get("hxyw") else None,
               str_dict.get("ywxq") if str_dict.get("ywxq") else None,
               str_dict.get("ywxt") if str_dict.get("ywxt") else None,
               str_dict.get("jscj") if str_dict.get("jscj") else None,
               str_dict.get("yhfw") if str_dict.get("yhfw") else None,
               str_dict.get("mbqt") if str_dict.get("mbqt") else None,
               str_dict.get("jsnr") if str_dict.get("jsnr") else None,
               str_dict.get("gnmk") if str_dict.get("gnmk") else None,
               str_dict.get("sjgx") if str_dict.get("sjgx") else None,
               str_dict.get("znys") if str_dict.get("znys") else None))
        # 或取所有的xmnr_copy1
        xmnr_copy1 = mysql.sql_select_many("""select * from user_history_data""")
        # 对比xmnr_copy1和xmnr维度是否都有
        if xmnr_copy1:
            for xc in xmnr_copy1:
                total_keywords = {}
                total_similarity = 0
                dup_count = 0
                # 保存相加后的相似度到idc_project_check
                mysql.sql_change_msg(
                    """insert into idc_project_check (project_id, dup_project_name, file_path, company_name, create_year, project_tag, project_range_tag, project_area, create_time, update_time) value ("%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s")"""
                    % (dl[0], xc.get("xmmc"), dl[1], "", "", "需求相似、业务相似", "历史项目", "",
                       str(datetime.datetime.now())[:-7], str(datetime.datetime.now())[:-7]))
                dup_id = mysql.cur.lastrowid
                for x in list(xc.keys())[1:]:
                    content_x = xc.get(x)
                    content_y = str_dict.get(x)
                    if content_x and content_y:
                        if x == 'gnmk':
                            continue
                        elif x == 'jsnr':
                            continue
                        else:
                            dup_count += 1
                if xc.get('gnmk')==' ' and str_dict.get('gnmk')==' ':
                    for x in list(xc.keys())[1:]:
                        content_x = xc.get(x)
                        content_y = str_dict.get(x)
                        if content_x and content_y:
                            if x == 'gnmk':
                                # 匹配到历史数据，次数加1
                                #     dup_count += dup_file_test
                                # 循环遍历每一个维度
                                similarity = cosin_similarity.CosineSimilarity(content_x, content_y)
                                # 相似度  关键词
                                similarity, keywords_x, keywords_y = similarity.main()
                                similarity = similarity * 0
                                #print("**************相似度: %.2f%%" % similarity, "关键词: %s" % keywords_y)
                                # 相似度相加
                                total_similarity += similarity
                                # 关键词收集
                                total_keywords[x] = keywords_y

                                function_content = content_y
                                dup_function_content = content_x
                                for word_y in keywords_y:
                                    word_y = word_y.strip().strip("'").strip('"')
                                    function_content = str(function_content.replace("\"", "'")).replace(word_y,
                                                                                                        f'<span class="similarity">{word_y.strip()}</span>')
                                for word_x in keywords_x:
                                    word_x = word_x.strip().strip("'").strip('"')
                                    dup_function_content = str(dup_function_content.replace("\"", "'")).replace(word_x,
                                                                                                                f'<span class="similarity">{word_x.strip()}</span>')
                                # 保存每个维度对应的相似度到idc_project_check_detail
                                mysql.sql_change_msg(
                                    """insert into idc_project_check_detail (dup_id, dimension, similarity, function_content, dup_function_content, create_time, update_time) value (%d, "%s", %f, "%s", "%s", "%s", "%s")"""
                                    % (dup_id, wdys2.get(x), similarity, escape_string(function_content),
                                       escape_string(dup_function_content), str(datetime.datetime.now())[:-7],
                                       str(datetime.datetime.now())[:-7]))
                                # content = content.replace(gjcs, f'<span class="similarity">{gjcs.strip()}</span>')
                            elif x == 'jsnr':
                                similarity = cosin_similarity.CosineSimilarity(content_x, content_y)
                                # 相似度  关键词
                                similarity, keywords_x, keywords_y = similarity.main()
                                similarity = similarity * 40
                                #print("**************相似度: %.2f%%" % similarity, "关键词: %s" % keywords_y)
                                # 相似度相加
                                total_similarity += similarity
                                # 关键词收集
                                total_keywords[x] = keywords_y
                                function_content = content_y
                                dup_function_content = content_x
                                for word_y in keywords_y:
                                    word_y = word_y.strip().strip("'").strip('"')
                                    function_content = str(function_content.replace("\"", "'")).replace(word_y,
                                                                                                        f'<span class="similarity">{word_y.strip()}</span>')
                                for word_x in keywords_x:
                                    word_x = word_x.strip().strip("'").strip('"')
                                    dup_function_content = str(dup_function_content.replace("\"", "'")).replace(word_x,
                                                                                                                f'<span class="similarity">{word_x.strip()}</span>')
                                # 保存每个维度对应的相似度到idc_project_check_detail
                                mysql.sql_change_msg(
                                    """insert into idc_project_check_detail (dup_id, dimension, similarity, function_content, dup_function_content, create_time, update_time) value (%d, "%s", %f, "%s", "%s", "%s", "%s")"""
                                    % (dup_id, wdys2.get(x), similarity, escape_string(function_content),
                                       escape_string(dup_function_content), str(datetime.datetime.now())[:-7],
                                       str(datetime.datetime.now())[:-7]))
                            else:
                                similarity = cosin_similarity.CosineSimilarity(content_x, content_y)
                                # 相似度  关键词
                                similarity, keywords_x, keywords_y = similarity.main()
                                similarity = similarity * (60 / dup_count)
                                #print("**************相似度: %.2f%%" % similarity, "关键词: %s" % keywords_y)
                                # 相似度相加
                                total_similarity += similarity
                                # 关键词收集
                                total_keywords[x] = keywords_y
                                function_content = content_y
                                dup_function_content = content_x

                                for word_y in keywords_y:
                                    word_y = word_y.strip().strip("'").strip('"')
                                    function_content = str(function_content.replace("\"", "'")).replace(word_y,
                                                                                                        f'<span class="similarity">{word_y.strip()}</span>')
                                for word_x in keywords_x:
                                    word_x = word_x.strip().strip("'").strip('"')
                                    dup_function_content = str(dup_function_content.replace("\"", "'")).replace(word_x,
                                                                                                                f'<span class="similarity">{word_x.strip()}</span>')
                                # 保存每个维度对应的相似度到idc_project_check_detail
                                mysql.sql_change_msg(
                                    """insert into idc_project_check_detail (dup_id, dimension, similarity, function_content, dup_function_content, create_time, update_time) value (%d, "%s", %f, "%s", "%s", "%s", "%s")"""
                                    % (dup_id, wdys2.get(x), similarity, escape_string(function_content),
                                       escape_string(dup_function_content), str(datetime.datetime.now())[:-7],
                                       str(datetime.datetime.now())[:-7]))
                elif xc['jsnr'] == ' ' and str_dict['jsnr'] == ' ':
                    for x in list(xc.keys())[1:]:
                        content_x = xc.get(x)
                        content_y = str_dict.get(x)
                        if content_x and content_y:
                            if x == 'gnmk':
                                # 匹配到历史数据，次数加1
                                #     dup_count += dup_file_test
                                # 循环遍历每一个维度
                                similarity = cosin_similarity.CosineSimilarity(content_x, content_y)
                                # 相似度  关键词
                                similarity, keywords_x, keywords_y = similarity.main()
                                similarity = similarity * 50
                                #print("**************相似度: %.2f%%" % similarity, "关键词: %s" % keywords_y)
                                # 相似度相加
                                total_similarity += similarity
                                # 关键词收集
                                total_keywords[x] = keywords_y

                                function_content = content_y
                                dup_function_content = content_x
                                for word_y in keywords_y:
                                    word_y = word_y.strip().strip("'").strip('"')
                                    function_content = str(function_content.replace("\"", "'")).replace(word_y,
                                                                                                        f'<span class="similarity">{word_y.strip()}</span>')
                                for word_x in keywords_x:
                                    word_x = word_x.strip().strip("'").strip('"')
                                    dup_function_content = str(dup_function_content.replace("\"", "'")).replace(word_x,
                                                                                                                f'<span class="similarity">{word_x.strip()}</span>')
                                # 保存每个维度对应的相似度到idc_project_check_detail
                                mysql.sql_change_msg(
                                    """insert into idc_project_check_detail (dup_id, dimension, similarity, function_content, dup_function_content, create_time, update_time) value (%d, "%s", %f, "%s", "%s", "%s", "%s")"""
                                    % (dup_id, wdys2.get(x), similarity, escape_string(function_content),
                                       escape_string(dup_function_content), str(datetime.datetime.now())[:-7],
                                       str(datetime.datetime.now())[:-7]))
                                # content = content.replace(gjcs, f'<span class="similarity">{gjcs.strip()}</span>')
                            elif x == 'jsnr':
                                similarity = cosin_similarity.CosineSimilarity(content_x, content_y)
                                # 相似度  关键词
                                similarity, keywords_x, keywords_y = similarity.main()
                                similarity = similarity * 0
                                #print("**************相似度: %.2f%%" % similarity, "关键词: %s" % keywords_y)
                                # 相似度相加
                                total_similarity += similarity
                                # 关键词收集
                                total_keywords[x] = keywords_y
                                function_content = content_y
                                dup_function_content = content_x
                                for word_y in keywords_y:
                                    word_y = word_y.strip().strip("'").strip('"')
                                    function_content = str(function_content.replace("\"", "'")).replace(word_y,
                                                                                                        f'<span class="similarity">{word_y.strip()}</span>')
                                for word_x in keywords_x:
                                    word_x = word_x.strip().strip("'").strip('"')
                                    dup_function_content = str(dup_function_content.replace("\"", "'")).replace(word_x,
                                                                                                                f'<span class="similarity">{word_x.strip()}</span>')
                                # 保存每个维度对应的相似度到idc_project_check_detail
                                mysql.sql_change_msg(
                                    """insert into idc_project_check_detail (dup_id, dimension, similarity, function_content, dup_function_content, create_time, update_time) value (%d, "%s", %f, "%s", "%s", "%s", "%s")"""
                                    % (dup_id, wdys2.get(x), similarity, escape_string(function_content),
                                       escape_string(dup_function_content), str(datetime.datetime.now())[:-7],
                                       str(datetime.datetime.now())[:-7]))
                            else:
                                similarity = cosin_similarity.CosineSimilarity(content_x, content_y)
                                # 相似度  关键词
                                similarity, keywords_x, keywords_y = similarity.main()
                                similarity = similarity * (50 / dup_count)
                                #print("**************相似度: %.2f%%" % similarity, "关键词: %s" % keywords_y)
                                # 相似度相加
                                total_similarity += similarity
                                # 关键词收集
                                total_keywords[x] = keywords_y
                                function_content = content_y
                                dup_function_content = content_x

                                for word_y in keywords_y:
                                    word_y = word_y.strip().strip("'").strip('"')
                                    function_content = str(function_content.replace("\"", "'")).replace(word_y,
                                                                                                        f'<span class="similarity">{word_y.strip()}</span>')
                                for word_x in keywords_x:
                                    word_x = word_x.strip().strip("'").strip('"')
                                    dup_function_content = str(dup_function_content.replace("\"", "'")).replace(word_x,
                                                                                                                f'<span class="similarity">{word_x.strip()}</span>')
                                # 保存每个维度对应的相似度到idc_project_check_detail
                                mysql.sql_change_msg(
                                    """insert into idc_project_check_detail (dup_id, dimension, similarity, function_content, dup_function_content, create_time, update_time) value (%d, "%s", %f, "%s", "%s", "%s", "%s")"""
                                    % (dup_id, wdys2.get(x), similarity, escape_string(function_content),
                                       escape_string(dup_function_content), str(datetime.datetime.now())[:-7],
                                       str(datetime.datetime.now())[:-7]))
                else:
                    for x in list(xc.keys())[1:]:
                        content_x = xc.get(x)
                        content_y = str_dict.get(x)
                        if content_x and content_y:
                            if x == 'gnmk':
                                # 匹配到历史数据，次数加1
                                #     dup_count += dup_file_test
                                # 循环遍历每一个维度
                                similarity = cosin_similarity.CosineSimilarity(content_x, content_y)
                                # 相似度  关键词
                                similarity, keywords_x, keywords_y = similarity.main()
                                similarity = similarity * 50
                                #print("**************相似度: %.2f%%" % similarity, "关键词: %s" % keywords_y)
                                # 相似度相加
                                total_similarity += similarity
                                # 关键词收集
                                total_keywords[x] = keywords_y

                                function_content = content_y
                                dup_function_content = content_x
                                for word_y in keywords_y:
                                    word_y = word_y.strip().strip("'").strip('"')
                                    function_content = str(function_content.replace("\"", "'")).replace(word_y,
                                                                                                        f'<span class="similarity">{word_y.strip()}</span>')
                                for word_x in keywords_x:
                                    word_x = word_x.strip().strip("'").strip('"')
                                    dup_function_content = str(dup_function_content.replace("\"", "'")).replace(word_x,
                                                                                                                f'<span class="similarity">{word_x.strip()}</span>')
                                # 保存每个维度对应的相似度到idc_project_check_detail
                                mysql.sql_change_msg(
                                    """insert into idc_project_check_detail (dup_id, dimension, similarity, function_content, dup_function_content, create_time, update_time) value (%d, "%s", %f, "%s", "%s", "%s", "%s")"""
                                    % (dup_id, wdys2.get(x), similarity, escape_string(function_content),
                                       escape_string(dup_function_content), str(datetime.datetime.now())[:-7],
                                       str(datetime.datetime.now())[:-7]))
                                # content = content.replace(gjcs, f'<span class="similarity">{gjcs.strip()}</span>')
                            elif x == 'jsnr':
                                similarity = cosin_similarity.CosineSimilarity(content_x, content_y)
                                # 相似度  关键词
                                similarity, keywords_x, keywords_y = similarity.main()
                                similarity = similarity * 40
                                #print("**************相似度: %.2f%%" % similarity, "关键词: %s" % keywords_y)
                                # 相似度相加
                                total_similarity += similarity
                                # 关键词收集
                                total_keywords[x] = keywords_y
                                function_content = content_y
                                dup_function_content = content_x
                                for word_y in keywords_y:
                                    word_y = word_y.strip().strip("'").strip('"')
                                    function_content = str(function_content.replace("\"", "'")).replace(word_y,
                                                                                                        f'<span class="similarity">{word_y.strip()}</span>')
                                for word_x in keywords_x:
                                    word_x = word_x.strip().strip("'").strip('"')
                                    dup_function_content = str(dup_function_content.replace("\"", "'")).replace(word_x,
                                                                                                                f'<span class="similarity">{word_x.strip()}</span>')
                                # 保存每个维度对应的相似度到idc_project_check_detail
                                mysql.sql_change_msg(
                                    """insert into idc_project_check_detail (dup_id, dimension, similarity, function_content, dup_function_content, create_time, update_time) value (%d, "%s", %f, "%s", "%s", "%s", "%s")"""
                                    % (dup_id, wdys2.get(x), similarity, escape_string(function_content),
                                       escape_string(dup_function_content), str(datetime.datetime.now())[:-7],
                                       str(datetime.datetime.now())[:-7]))
                            else:
                                similarity = cosin_similarity.CosineSimilarity(content_x, content_y)
                                # 相似度  关键词
                                similarity, keywords_x, keywords_y = similarity.main()
                                similarity = similarity * (10 / dup_count)
                                #print("**************相似度: %.2f%%" % similarity, "关键词: %s" % keywords_y)
                                # 相似度相加
                                total_similarity += similarity
                                # 关键词收集
                                total_keywords[x] = keywords_y
                                function_content = content_y
                                dup_function_content = content_x

                                for word_y in keywords_y:
                                    word_y = word_y.strip().strip("'").strip('"')
                                    function_content = str(function_content.replace("\"", "'")).replace(word_y,
                                                                                                        f'<span class="similarity">{word_y.strip()}</span>')
                                for word_x in keywords_x:
                                    word_x = word_x.strip().strip("'").strip('"')
                                    dup_function_content = str(dup_function_content.replace("\"", "'")).replace(word_x,
                                                                                                                f'<span class="similarity">{word_x.strip()}</span>')
                                # 保存每个维度对应的相似度到idc_project_check_detail
                                mysql.sql_change_msg(
                                    """insert into idc_project_check_detail (dup_id, dimension, similarity, function_content, dup_function_content, create_time, update_time) value (%d, "%s", %f, "%s", "%s", "%s", "%s")"""
                                    % (dup_id, wdys2.get(x), similarity, escape_string(function_content),
                                       escape_string(dup_function_content), str(datetime.datetime.now())[:-7],
                                       str(datetime.datetime.now())[:-7]))

                mysql.sql_change_msg(
                    """update idc_project_check set similarity=%f where dup_id=%d""" % (total_similarity, dup_id))
            project_gjc = {}
            for w in wdys2.keys():
                content_x = str_dict.get(w)
                content_y = str_dict.get(w)
                if content_x and content_y:
                    # 循环遍历每一个维度
                    similarity = cosin_similarity.CosineSimilarity(content_x, content_y)
                    # 相似度  关键词
                    similarity, keywords_x, keywords = similarity.main()
                    project_gjc[w] = keywords
            mysql.sql_change_msg(
                """insert into user_keyword (xmmc, xzwt, xtjc, xmmb, yqjx, jsxq, sjxq, aqxq, ywly, hxyw, ywxq, ywxt, jscj, yhfw, mbqt, jsnr, gnmk, sjgx, znys) value ("%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s")"""
                % (dl[0], str(project_gjc.get("xzwt"))[1:-1] if project_gjc.get("xzwt") else None,
                   str(project_gjc.get("xtjc"))[1:-1] if project_gjc.get("xtjc") else None,
                   str(project_gjc.get("xmmb"))[1:-1] if project_gjc.get("xmmb") else None,
                   str(project_gjc.get("yqjx"))[1:-1] if project_gjc.get("yqjx") else None,
                   str(project_gjc.get("jsxq"))[1:-1] if project_gjc.get("jsxq") else None,
                   str(project_gjc.get("sjxq"))[1:-1] if project_gjc.get("sjxq") else None,
                   str(project_gjc.get("aqxq"))[1:-1] if project_gjc.get("aqxq") else None,
                   str(project_gjc.get("ywly"))[1:-1] if project_gjc.get("ywly") else None,
                   str(project_gjc.get("hxyw"))[1:-1] if project_gjc.get("hxyw") else None,
                   str(project_gjc.get("ywxq"))[1:-1] if project_gjc.get("ywxq") else None,
                   str(project_gjc.get("ywxt"))[1:-1] if project_gjc.get("ywxt") else None,
                   str(project_gjc.get("jscj"))[1:-1] if project_gjc.get("jscj") else None,
                   str(project_gjc.get("yhfw"))[1:-1] if project_gjc.get("yhfw") else None,
                   str(project_gjc.get("mbqt"))[1:-1] if project_gjc.get("mbqt") else None,
                   str(project_gjc.get("jsnr"))[1:-1] if project_gjc.get("jsnr") else None,
                   str(project_gjc.get("gnmk"))[1:-1] if project_gjc.get("gnmk") else None,
                   str(project_gjc.get("sjgx"))[1:-1] if project_gjc.get("sjgx") else None,
                   str(project_gjc.get("znys"))[1:-1] if project_gjc.get("znys") else None))
        mysql.sql_change_msg(
            """update idc_project set dup_status=3, one_vote_veto_status=dup_file_test, self_check_status=dup_file_test, history_project_count=%d ,module_count=%d where project_id=%d""" % (
                xmnr_count, gnmk_count, dl[0]))
        gong_neng_mo_kuai(mysql, dl, data, er_title)


if __name__ == "__main__":
    # all_path = requests.get("http://127.0.0.1:19099/check/duplicates/%s" % 15).json()
    # print(all_path)

    # data_list = []
    # for ap in all_path.get("data"):
    #     # if os.path.exists(ap.get("file_path")):
    #         data_list.append((ap.get("project_id"), ap.get("file_path"), ap.get("project_name")))
    # print(data_list)
    data_list = [(11, r"C:\Users\HUAWEI\PycharmProjects\nlp\dup_check\0825-丽水系统查重维度1.xlsx", "水路运输综合监管系统建设项目.xls")]
    project_check(data_list)
"""

"""