"""
理解sklearn中的CountVectorizer和TfidfVectorizer
"""
from collections import Counter
import numpy as np
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
sentences = ["there is a dog dog", "here is a cat"]
count_vec = CountVectorizer()
a = count_vec.fit_transform(sentences)
print(a.toarray())
print(count_vec.vocabulary_)
"""
输出
{'dog': 1, 'there': 4, 'here': 2, 'cat': 0, 'is': 3}
表示每个词汇对应的坐标
"""
print("=" * 10)
tf_vec = TfidfVectorizer()
b = tf_vec.fit_transform(sentences)
print(b.toarray())
print(tf_vec.vocabulary_)
print(tf_vec.idf_) # 逆文档频率
print(tf_vec.get_feature_names())
def mytf_idf(s):
# 自己实现tfidf
words = tf_vec.get_feature_names()
tf_matrix = np.zeros((len(s), len(words)), dtype=np.float32)
smooth = 1
# 初始值加上平滑因子
df_matrix = np.ones(len(words), dtype=np.float32) * smooth
for i in range(len(s)):
s_words = s[i].split()
for j in range(len(words)):
cnt = Counter(s_words).get(words[j], 0)
tf_matrix[i][j] = cnt
if cnt > 0:
df_matrix[j] += 1
# idf一定是大于1的数值
idf_matrix = np.log((len(s) + smooth) / df_matrix) + 1
matrix = tf_matrix * idf_matrix
matrix = matrix / np.linalg.norm(matrix, 2, axis=1).reshape(matrix.shape[0], 1)
print(matrix)
print("=" * 10)
mytf_idf(sentences)
"""
TODO:
* IDF可以学到，通过神经网络反向传播来学习IDF而不是直接计算得出
* CountVectorizer有时不需要考虑个数，只需要知道是否出现过即可
"""
