# coding: utf-8
#本代码主要实现了对于商品名称根据tfidf提取特征词,然后基于已经训练好的word2vec模型,对每行商品的tfidf值大于某一阈值的特征词相似度匹配已经给定的商品类别。
import jieba
import jieba.posseg as pseg
import jieba.analyse
import pymssql
import xlwt
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, TfidfTransformer
import pandas as pd
jieba.load_userdict('C:\\Users\\Desktop\\s_proj\\dict.txt') #导入自定义的分词词典
#连接数据库
conn = pymssql.connect(host='1.1.1.1',user='username',password='password',database='database',charset='utf8')
cur = conn.cursor()
sql='select distinct(column) from table'
cur.execute(sql)
listl=cur.fetchall()
#分词
words=[]
for word in listl:
for i in word:
seg_list = jieba.cut(i, cut_all=False)
words.append(" ".join(seg_list))
#计算tfidf
tfidf_vectorizer = CountVectorizer(min_df=2,max_df = 0.8,token_pattern=u'(?u)\\b[^\\d\\W]\\w+\\b')
transformer=TfidfTransformer()
word_tfidf = tfidf_vectorizer.fit_transform(words)
tfidf=transformer.fit_transform(word_tfidf)
print(tfidf.shape)
#获取特征词
features=tfidf_vectorizer.get_feature_names()
#加载已训练好的word2vec模型
from gensim.models import word2vec
import logging
logging.basicConfig(format='%(asctime)s:%(levelname)s: %(message)s', level=logging.INFO)
model=gensim.models.KeyedVectors.load_word2vec_format('model_word.bin',binary=True)
#打开要匹配相似度的商品类文本
f4=open('C:\\Users\\Desktop\\s_proj\\c.txt','r',encoding='utf-8',errors='ignore')
#对商品类文本做文本处理
ff=[]
for j in f4.readlines():
j=j.replace('\n','')
ff.append(j)
print(len(ff))
#tfidf 稀疏矩阵
tfidf_tuple = tfidf.nonzero()
tfidf_rows = tfidf_tuple[0]
tfidf_columns = tfidf_tuple[1]
size = len(tfidf_columns)
print('nonzero.size=%s' % size)
# 将TF-IDF>=某一阈值d的数据存入字典,Key为电商索引。
product_dict = {}
for i in range(size):
row = tfidf_rows[i]
column = tfidf_columns[i]
tfidf_value = tfidf[row,column]
if tfidf_value <=0.4:
continue
key_words = product_dict.setdefault(row, [])
key_word = {}
key_word["key_word"]=features[column]
key_word["tfidf_value"]=tfidf_value
key_words.append(key_word)
print('product_dict.len=%s' % len(product_dict))
#tfidf提取的所有features计算word2vec相似度
f=open('gabbage.txt','w',encoding='utf-8',errors='ignore')
word2vec={}
for i in range(size):
column = tfidf_columns[i]
cate=word2vec.setdefault(features[column],[])
for jj in ff:
try:
y1 = model.wv.similarity(features[column],jj)
insert={"category_name":jj,"similarity":y1}
cate.append(insert)
except:
f.write('')
print('word2vec.len=%s' % len(word2vec))
f.close()
#相似度值排序
import operator
for k,v in word2vec.items():
new_dict_list=[]
s=sorted(v,key=operator.itemgetter("similarity"),reverse=True)
word2vec[k]=s
for w in word2vec[k]:
if w not in new_dict_list:
new_dict_list.append(w)
word2vec[k]=new_dict_list
#创建满足某一阈值的tfidf的特征词的相似度的词典
l=[ll for l in product_dict.keys()for ll in product_dict[l]]
for k in word2vec.keys():
for m in l:
if m["key_word"]==k:
m["category_names"]=word2vec[k]
print(len(product_dict))
# 将字典转为数组,索引转为商品名称。
result=[r for r in listl]
product_list = []
for i in product_dict:
product = {}
product['product'] = result[i][0]
product['key_words'] = product_dict[i]
product_list.append(product)
print('product_list.len=%s' % len(product_list))
print(len(product_list))
#匹配填充tfidf值满足某一阈值的相似度词典
f=open('gabbage.txt','w',encoding='utf-8',errors='ignore')
# 将TF-IDF>=0。4的数据存入字典,Key为电商索引。
product_dict = {}
for i in range(size):
row = tfidf_rows[i]
column = tfidf_columns[i]
tfidf_value = tfidf[row,column]
if tfidf_value <=0.4:
continue
key_words = product_dict.setdefault(row, [])
for jj in ff:
try:
y1 = model.wv.similarity(features[column],jj)
insert={'key_word':features[column],'tfidf_value':tfidf_value,'category_names':{'goods_name':jj,'word2vec':y1}}
key_words.append(insert)
except:
f.write('')
f.close()
#将结果写入json
import json
t=open('word2vec_result1.txt','w',encoding='utf-8',errors='ignore')
json_string = json.dumps(product_list[0:11]).encode('utf-8').decode('unicode_escape')
print(json_string)
t.write(json_string)
t.close()