huangxiyun
/
dup_check
1
0
Форк 0
Код Проблеми 0 Запити на злиття 0 Релізи 0 Вікі Активність
Ви не можете вибрати більше 25 тем Теми мають розпочинатися з літери або цифри, можуть містити дефіси (-) і не повинні перевищувати 35 символів.
1 Коміт
1 Гілка
92KB
Дерево: d9ffa090a9
Гілки Теги
${ item.name }
Створити гілку ${ searchTerm }
з 'd9ffa090a9'
${ noResults }
dup_check/main10.py

392 рядки
22KB
Неформатований Звинувачення Історія 

  1. # coding=utf-8

  2. import sys

  3. import re

  4. import mysql_pool

  5. from pymysql.converters import escape_string

  6. import cosin_similarity

  7. import pandas as pd

  8. import datetime

  9. import requests

  10. import os

  11. 

  12. wdys1 = {

  13.     "项目名称": "xmmc",

  14.     "现状问题": "xzwt",

  15.     "系统基础": "xtjc",

  16.     "项目目标": "xmmb",

  17.     "预期绩效": "yqjx",

  18.     "建设需求": "jsxq",

  19.     "数据需求": "sjxq",

  20.     "安全需求": "aqxq",

  21.     "业务领域": "ywly",

  22.     "核心业务": "hxyw",

  23.     "业务需求": "ywxq",

  24.     "业务协同": "ywxt",

  25.     "建设层级": "jscj",

  26.     "用户范围": "yhfw",

  27.     "目标群体": "mbqt",

  28.     "建设内容": "jsnr",

  29.     "功能模块": "gnmk",

  30.     "数据共享": "sjgx",

  31.     "智能要素": "znys"

  32. }

  33. wdys2 = {

  34.     "xmmc": "项目名称",

  35.     "xzwt": "现状问题",

  36.     "xtjc": "系统基础",

  37.     "xmmb": "项目目标",

  38.     "yqjx": "预期绩效",

  39.     "jsxq": "建设需求",

  40.     "sjxq": "数据需求",

  41.     "aqxq": "安全需求",

  42.     "ywly": "业务领域",

  43.     "hxyw": "核心业务",

  44.     "ywxq": "业务需求",

  45.     "ywxt": "业务协同",

  46.     "jscj": "建设层级",

  47.     "yhfw": "用户范围",

  48.     "mbqt": "目标群体",

  49.     "jsnr": "建设内容",

  50.     "gnmk": "功能模块",

  51.     "sjgx": "数据共享",

  52.     "znys": "智能要素"

  53. }

  54. gnmkys = {

  55.     "gnmc": "功能名称",

  56.     "gnms": "功能描述"

  57. }

  58. 

  59. 

  60. def getFlag():

  61.     data_dict = {}

  62.     df = pd.read_excel("0825-丽水系统查重维度.xlsx")

  63.     data = df.values

  64.     data = list(pd.Series(data[:, 1]).dropna())

  65.     for d in data:

  66.         try:

  67.             wd = re.search("(.*?)（.*?%）", d).group(1).strip()

  68.             wdc = wdys1.get(wd)

  69.             if wdc:

  70.                 qz = re.search(".*?（(.*?%)）", d).group(1)

  71.                 data_dict[wdc] = qz

  72.         except:

  73.             pass

  74.     return data_dict

  75. 

  76. 

  77. def gong_neng_mo_kuai(mysql, dl, data, er_title):

  78.     # 将excel文件中的所有第三维度内容进行拼接

  79.     str_dict = {}

  80.     for et in er_title:

  81.         for d in data:

  82.             if d[1] == et:

  83.                 if str_dict.get(et):

  84.                     str_dict[et] = str_dict.get(et) + d[3]

  85.                 else:

  86.                     str_dict[et] = d[3]

  87.     # print(str_dict)

  88.     for k, v in str_dict.items():

  89.         mysql.sql_change_msg(

  90.             """insert into idc_project_module (project_id, check_duplicate_count, module_name, module_content, create_time, update_time, tag) value(%d, 1, "%s", "%s", "%s", "%s", "模块")""" % (

  91.                 int(dl[0]), k, v, str(datetime.datetime.now())[:-7], str(datetime.datetime.now())[:-7]))

  92.     module_id_list = mysql.sql_select_many(

  93.         """select project_module_id, module_name, module_content  from idc_project_module where project_id=%d""" % dl[

  94.             0])

  95.     data_list = []

  96.     for mil in module_id_list:

  97.         data_dict = {}

  98.         data_dict["project_module_id"] = mil.get("project_module_id")

  99.         data_dict["gnmc"] = mil.get("module_name")

  100.         data_dict["gnms"] = mil.get("module_content")

  101.         data_list.append(data_dict)

  102.     # print(data_list)

  103.     for i in data_list:

  104.         gnmk_copy1 = mysql.sql_select_many("""select * from user_history_module_data""")

  105.         if gnmk_copy1:

  106.             for gc in gnmk_copy1:

  107.                 total_similarity1 = 0

  108.                 total_keywords1 = []

  109.                 total_similarity2 = 0

  110.                 total_keywords2 = []

  111.                 mysql.sql_change_msg(

  112.                     """insert into idc_project_module_check (project_module_id, module_name, project_name, company_name, create_time, update_time) value(%d, "%s", "%s", "%s", "%s", "%s")"""

  113.                     % (

  114.                         i.get("project_module_id"), gc.get("gnmc"), gc.get("xmmc"), "",

  115.                         str(datetime.datetime.now())[:-7],

  116.                         str(datetime.datetime.now())[:-7]))

  117.                 dup_module_id = mysql.cur.lastrowid

  118.                 for j in ["gnmc", "gnms"]:

  119.                     # 循环遍历每一个模块名称

  120.                     content_x = gc.get(j)

  121.                     content_y = i.get(j)

  122.                     if content_x and content_y:

  123.                         if j == "gnmc":

  124.                             similarity = cosin_similarity.CosineSimilarity(content_x, content_y)

  125.                             # 相似度  关键词

  126.                             similarity, keyword_x, keywords = similarity.main()

  127.                             similarity = similarity * 1

  128.                             total_keywords1 += keywords

  129.                             print("######################相似度: %.2f%%" % similarity, "关键词: %s" % keywords)

  130.                             # 相似度相加

  131.                             total_similarity1 += similarity

  132.                             mysql.sql_change_msg(

  133.                                 """insert into idc_project_module_check_detail (dup_module_id, project_name, module_content, dup_module_content, similarity, dimension, create_time, update_time) value (%d, "%s", "%s", "%s", %f, "%s", "%s", "%s")"""

  134.                                 % (dup_module_id, dl[2], escape_string(content_y), escape_string(content_x), similarity,

  135.                                    "功能名称",

  136.                                    str(datetime.datetime.now())[:-7], str(datetime.datetime.now())[:-7]))

  137.                         else:

  138.                             similarity = cosin_similarity.CosineSimilarity(content_x, content_y)

  139.                             # 相似度  关键词

  140.                             similarity, keyword_x, keywords = similarity.main()

  141.                             similarity = similarity * 99

  142.                             total_keywords2 += keywords

  143.                             print("######################相似度: %.2f%%" % similarity, "关键词: %s" % keywords)

  144.                             # 相似度相加

  145.                             total_similarity2 += similarity

  146.                             mysql.sql_change_msg(

  147.                                 """insert into idc_project_module_check_detail (dup_module_id, project_name, module_content, dup_module_content, similarity, dimension, create_time, update_time) value (%d, "%s", "%s", "%s", %f, "%s", "%s", "%s")"""

  148.                                 % (dup_module_id, dl[2], escape_string(content_y), escape_string(content_x), similarity,

  149.                                    "功能模块描述",

  150.                                    str(datetime.datetime.now())[:-7], str(datetime.datetime.now())[:-7]))

  151.                 mysql.sql_change_msg("""update idc_project_module_check set similarity=%f where dup_module_id=%d""" % (

  152.                     total_similarity1 + total_similarity2, dup_module_id))

  153.             gnmk_gjc = {}

  154.             for a in ["gnmc", "gnms"]:

  155.                 if i.get(a):

  156.                     content_x = i.get(a)

  157.                     content_y = i.get(a)

  158.                     if a == "gnmc":

  159.                         similarity = cosin_similarity.CosineSimilarity(content_x, content_y)

  160.                         # 相似度  关键词

  161.                         similarity, keyword_x, keywords = similarity.main()

  162.                         gnmk_gjc[a] = keywords

  163.                     else:

  164.                         similarity = cosin_similarity.CosineSimilarity(content_x, content_y)

  165.                         # 相似度  关键词

  166.                         similarity, keyword_x, keywords = similarity.main()

  167.                         gnmk_gjc[a] = keywords

  168.             mysql.sql_change_msg("""insert into user_module_keywords (xmmc, gnmc, gnms) value("%s", "%s", "%s")""" % (

  169.                 dl[2], str(gnmk_gjc.get("gnmc"))[1:-1] if gnmk_gjc.get("gnmc") else None,

  170.                 str(gnmk_gjc.get("gnms"))[1:-1] if gnmk_gjc.get("gnms") else None))

  171. 

  172. 

  173. def project_check(data_list):

  174.     mysql = mysql_pool.ConnMysql()

  175.     # mysql.sql_select_many("""select * from mkgjc""")

  176.     # 读取维度和权重

  177.     # xmnr_count = len(mysql.sql_select_many("""select * from xmnr_copy1"""))

  178.     # gnmk_count = len(mysql.sql_select_many("""select * from gnmk_copy1"""))

  179.     xmnr_count = len(mysql.sql_select_many("""select * from user_history_data"""))

  180.     gnmk_count = len(mysql.sql_select_many("""select * from user_history_module_data"""))

  181.     get_data_dict = getFlag()

  182.     # 遍历excel存储路径

  183.     for dl in data_list:

  184.         # path = "0825-丽水系统查重维度1.xlsx"

  185.         # 读取路径下的excel

  186.         df = pd.read_excel(dl[1])

  187.         data = df.values

  188.         # 将excel文件中的所有维度内容进行拼接

  189.         join_str = ""

  190.         str_dict = {}

  191.         title = ""

  192.         er_title = set()

  193.         for d in data:

  194.             if pd.notnull(d[0]):

  195.                 title = d[0]

  196.                 if title == "功能模块":

  197.                     er_title.add(d[1])

  198.                 join_str = ""

  199.                 for i in d[1:]:

  200.                     if pd.notnull(i):

  201.                         join_str +=i

  202.                 str_dict[wdys1.get(title)] = join_str

  203.             else:

  204.                 if title == "功能模块":

  205.                     er_title.add(d[1])

  206.                 for i in d[1:]:

  207.                     if pd.notnull(i):

  208.                         join_str +=i

  209.                 str_dict[wdys1.get(title)] = str_dict.get(wdys1.get(title)) + join_str

  210.         # print(str_dict)

  211.         mysql.sql_change_msg(

  212.             """insert into user_data (xmmc, xzwt, xtjc, xmmb, yqjx, jsxq, sjxq, aqxq, ywly, hxyw, ywxq, ywxt, jscj, yhfw, mbqt, jsnr, gnmk, sjgx, znys) value ("%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s")"""

  213.             % (dl[0], str_dict.get("xzwt") if str_dict.get("xzwt") else None,

  214.                str_dict.get("xtjc") if str_dict.get("xtjc") else None,

  215.                str_dict.get("xmmb") if str_dict.get("xmmb") else None,

  216.                str_dict.get("yqjx") if str_dict.get("yqjx") else None,

  217.                str_dict.get("jsxq") if str_dict.get("jsxq") else None,

  218.                str_dict.get("sjxq") if str_dict.get("sjxq") else None,

  219.                str_dict.get("aqxq") if str_dict.get("aqxq") else None,

  220.                str_dict.get("ywly") if str_dict.get("ywly") else None,

  221.                str_dict.get("hxyw") if str_dict.get("hxyw") else None,

  222.                str_dict.get("ywxq") if str_dict.get("ywxq") else None,

  223.                str_dict.get("ywxt") if str_dict.get("ywxt") else None,

  224.                str_dict.get("jscj") if str_dict.get("jscj") else None,

  225.                str_dict.get("yhfw") if str_dict.get("yhfw") else None,

  226.                str_dict.get("mbqt") if str_dict.get("mbqt") else None,

  227.                str_dict.get("jsnr") if str_dict.get("jsnr") else None,

  228.                str_dict.get("gnmk") if str_dict.get("gnmk") else None,

  229.                str_dict.get("sjgx") if str_dict.get("sjgx") else None,

  230.                str_dict.get("znys") if str_dict.get("znys") else None))

  231.         # 或取所有的xmnr_copy1

  232.         xmnr_copy1 = mysql.sql_select_many("""select * from user_history_data""")

  233.         # 对比xmnr_copy1和xmnr维度是否都有

  234.         if xmnr_copy1:

  235.             for xc in xmnr_copy1:

  236.                 total_keywords = {}

  237.                 total_similarity = 0

  238.                 dup_count = 0

  239.                 # 保存相加后的相似度到idc_project_check

  240.                 mysql.sql_change_msg(

  241.                     """insert into idc_project_check (project_id, dup_project_name, file_path, company_name, create_year, project_tag, project_range_tag, project_area, create_time, update_time) value ("%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s")"""

  242.                     % (dl[0], xc.get("xmmc"), dl[1], "", "", "需求相似、业务相似", "历史项目", "",

  243.                        str(datetime.datetime.now())[:-7], str(datetime.datetime.now())[:-7]))

  244.                 dup_id = mysql.cur.lastrowid

  245.                 for x in list(xc.keys())[1:]:

  246.                     content_x = xc.get(x)

  247.                     content_y = str_dict.get(x)

  248.                     if content_x and content_y:

  249.                         if x == 'gnmk':

  250.                             continue

  251.                         elif x == 'jsnr':

  252.                             continue

  253.                         else:

  254.                             dup_count += 1

  255.                 for x in list(xc.keys())[1:]:

  256.                     content_x = xc.get(x)

  257.                     content_y = str_dict.get(x)

  258.                     if content_x and content_y:

  259.                         if x == 'gnmk':

  260.                             # 匹配到历史数据，次数加1

  261.                             #     dup_count += 1

  262.                             # 循环遍历每一个维度

  263.                             similarity = cosin_similarity.CosineSimilarity(content_x, content_y)

  264.                             # 相似度  关键词

  265.                             similarity, keywords_x, keywords_y = similarity.main()

  266.                             similarity = similarity * 50

  267.                             print("**************相似度: %.2f%%" % similarity, "关键词: %s" % keywords_y)

  268.                             # 相似度相加

  269.                             total_similarity += similarity

  270.                             # 关键词收集

  271.                             total_keywords[x] = keywords_y

  272. 

  273.                             function_content = content_y

  274.                             dup_function_content = content_x

  275.                             for word_y in keywords_y:

  276.                                 word_y = word_y.strip().strip("'").strip('"')

  277.                                 function_content = str(function_content.replace("\"", "'")).replace(word_y,

  278.                                                                                                     f'<span class="similarity">{word_y.strip()}</span>')

  279.                             for word_x in keywords_x:

  280.                                 word_x = word_x.strip().strip("'").strip('"')

  281.                                 dup_function_content = str(dup_function_content.replace("\"", "'")).replace(word_x,

  282.                                                                                                             f'<span class="similarity">{word_x.strip()}</span>')

  283.                             # 保存每个维度对应的相似度到idc_project_check_detail

  284.                             mysql.sql_change_msg(

  285.                                 """insert into idc_project_check_detail (dup_id, dimension, similarity, function_content, dup_function_content, create_time, update_time) value (%d, "%s", %f, "%s", "%s", "%s", "%s")"""

  286.                                 % (dup_id, wdys2.get(x), similarity, escape_string(function_content),

  287.                                    escape_string(dup_function_content), str(datetime.datetime.now())[:-7],

  288.                                    str(datetime.datetime.now())[:-7]))

  289.                             # content = content.replace(gjcs, f'<span class="similarity">{gjcs.strip()}</span>')

  290.                         elif x == 'jsnr':

  291.                             similarity = cosin_similarity.CosineSimilarity(content_x, content_y)

  292.                             # 相似度  关键词

  293.                             similarity, keywords_x, keywords_y = similarity.main()

  294.                             similarity = similarity * 40

  295.                             print("**************相似度: %.2f%%" % similarity, "关键词: %s" % keywords_y)

  296.                             # 相似度相加

  297.                             total_similarity += similarity

  298.                             # 关键词收集

  299.                             total_keywords[x] = keywords_y

  300.                             function_content = content_y

  301.                             dup_function_content = content_x

  302.                             for word_y in keywords_y:

  303.                                 word_y = word_y.strip().strip("'").strip('"')

  304.                                 function_content = str(function_content.replace("\"", "'")).replace(word_y,

  305.                                                                                                     f'<span class="similarity">{word_y.strip()}</span>')

  306.                             for word_x in keywords_x:

  307.                                 word_x = word_x.strip().strip("'").strip('"')

  308.                                 dup_function_content = str(dup_function_content.replace("\"", "'")).replace(word_x,

  309.                                                                                                             f'<span class="similarity">{word_x.strip()}</span>')

  310.                             # 保存每个维度对应的相似度到idc_project_check_detail

  311.                             mysql.sql_change_msg(

  312.                                 """insert into idc_project_check_detail (dup_id, dimension, similarity, function_content, dup_function_content, create_time, update_time) value (%d, "%s", %f, "%s", "%s", "%s", "%s")"""

  313.                                 % (dup_id, wdys2.get(x), similarity, escape_string(function_content),

  314.                                    escape_string(dup_function_content), str(datetime.datetime.now())[:-7],

  315.                                    str(datetime.datetime.now())[:-7]))

  316.                         else:

  317.                             similarity = cosin_similarity.CosineSimilarity(content_x, content_y)

  318.                             # 相似度  关键词

  319.                             similarity, keywords_x, keywords_y = similarity.main()

  320.                             similarity = similarity * (10 / dup_count)

  321.                             print("**************相似度: %.2f%%" % similarity, "关键词: %s" % keywords_y)

  322.                             # 相似度相加

  323.                             total_similarity += similarity

  324.                             # 关键词收集

  325.                             total_keywords[x] = keywords_y

  326.                             function_content = content_y

  327.                             dup_function_content = content_x

  328.                             for word_y in keywords_y:

  329.                                 word_y = word_y.strip().strip("'").strip('"')

  330.                                 function_content = str(function_content.replace("\"", "'")).replace(word_y,

  331.                                                                                                     f'<span class="similarity">{word_y.strip()}</span>')

  332.                             for word_x in keywords_x:

  333.                                 word_x = word_x.strip().strip("'").strip('"')

  334.                                 dup_function_content = str(dup_function_content.replace("\"", "'")).replace(word_x,

  335.                                                                                                             f'<span class="similarity">{word_x.strip()}</span>')

  336.                             # 保存每个维度对应的相似度到idc_project_check_detail

  337.                             mysql.sql_change_msg(

  338.                                 """insert into idc_project_check_detail (dup_id, dimension, similarity, function_content, dup_function_content, create_time, update_time) value (%d, "%s", %f, "%s", "%s", "%s", "%s")"""

  339.                                 % (dup_id, wdys2.get(x), similarity, escape_string(function_content),

  340.                                    escape_string(dup_function_content), str(datetime.datetime.now())[:-7],

  341.                                    str(datetime.datetime.now())[:-7]))

  342. 

  343.                 mysql.sql_change_msg(

  344.                     """update idc_project_check set similarity=%f where dup_id=%d""" % (total_similarity, dup_id))

  345.             project_gjc = {}

  346.             for w in wdys2.keys():

  347.                 content_x = str_dict.get(w)

  348.                 content_y = str_dict.get(w)

  349.                 if content_x and content_y:

  350.                     # 循环遍历每一个维度

  351.                     similarity = cosin_similarity.CosineSimilarity(content_x, content_y)

  352.                     # 相似度  关键词

  353.                     similarity, keywords_x, keywords = similarity.main()

  354.                     project_gjc[w] = keywords

  355.             mysql.sql_change_msg(

  356.                 """insert into user_keyword (xmmc, xzwt, xtjc, xmmb, yqjx, jsxq, sjxq, aqxq, ywly, hxyw, ywxq, ywxt, jscj, yhfw, mbqt, jsnr, gnmk, sjgx, znys) value ("%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s", "%s")"""

  357.                 % (dl[0], str(project_gjc.get("xzwt"))[1:-1] if project_gjc.get("xzwt") else None,

  358.                    str(project_gjc.get("xtjc"))[1:-1] if project_gjc.get("xtjc") else None,

  359.                    str(project_gjc.get("xmmb"))[1:-1] if project_gjc.get("xmmb") else None,

  360.                    str(project_gjc.get("yqjx"))[1:-1] if project_gjc.get("yqjx") else None,

  361.                    str(project_gjc.get("jsxq"))[1:-1] if project_gjc.get("jsxq") else None,

  362.                    str(project_gjc.get("sjxq"))[1:-1] if project_gjc.get("sjxq") else None,

  363.                    str(project_gjc.get("aqxq"))[1:-1] if project_gjc.get("aqxq") else None,

  364.                    str(project_gjc.get("ywly"))[1:-1] if project_gjc.get("ywly") else None,

  365.                    str(project_gjc.get("hxyw"))[1:-1] if project_gjc.get("hxyw") else None,

  366.                    str(project_gjc.get("ywxq"))[1:-1] if project_gjc.get("ywxq") else None,

  367.                    str(project_gjc.get("ywxt"))[1:-1] if project_gjc.get("ywxt") else None,

  368.                    str(project_gjc.get("jscj"))[1:-1] if project_gjc.get("jscj") else None,

  369.                    str(project_gjc.get("yhfw"))[1:-1] if project_gjc.get("yhfw") else None,

  370.                    str(project_gjc.get("mbqt"))[1:-1] if project_gjc.get("mbqt") else None,

  371.                    str(project_gjc.get("jsnr"))[1:-1] if project_gjc.get("jsnr") else None,

  372.                    str(project_gjc.get("gnmk"))[1:-1] if project_gjc.get("gnmk") else None,

  373.                    str(project_gjc.get("sjgx"))[1:-1] if project_gjc.get("sjgx") else None,

  374.                    str(project_gjc.get("znys"))[1:-1] if project_gjc.get("znys") else None))

  375.         mysql.sql_change_msg(

  376.             """update idc_project set dup_status=3, one_vote_veto_status=1, self_check_status=1, history_project_count=%d ,module_count=%d where project_id=%d""" % (

  377.                 xmnr_count, gnmk_count, dl[0]))

  378.         gong_neng_mo_kuai(mysql, dl, data, er_title)

  379. 

  380. 

  381. if __name__ == "__main__":

  382.     all_path = requests.get("http://127.0.0.1:19099/check/duplicates/%s" % 15).json()

  383.     # print(all_path)

  384.     #

  385.     # data_list = []

  386.     # for ap in all_path.get("data"):

  387.     #     # if os.path.exists(ap.get("file_path")):

  388.     #         data_list.append((ap.get("project_id"), ap.get("file_path"), ap.get("project_name")))

  389.     # print(data_list)

  390.     # data_list = [(11, r"D:\pythonDM\Ndkj\duplicate_check\0825-丽水系统查重维度1.xlsx", "数字百山祖（一期）—“云值守”建设方案")]

  391.     # project_check(data_list)