使用spacy检测文本相似度

# 这个比我们自己通过余弦相似度计算更精确一些，可以基本达到语义相似匹配
# This looks like it's the formula for computing cosine similarity and the vectors seem to be created with SpaCy's .vector which the documentation says is trained from GloVe's w2v model.
# github: https://github.com/explosion/spaCy
# doc： https://spacy.io/usage/vectors-similarity
# install: pip install spacy
# install english model: python -m spacy download en_core_web_sm[md|lg], ref: https://spacy.io/models/en
# install chinese model: python -m spacy download zh_core_web_sm[md|lg], ref: https://spacy.io/models/zh
import spacy
def check_word_property():
nlp = spacy.load("en_core_web_sm")
input_sentence = input('Please input a sentence: ')
while input_sentence:
doc = nlp(input_sentence)
word_property = [{'word': w.text, 'property': w.pos_} for w in doc]
noun_list = [wp['word'] for wp in word_property if wp['property'] == 'NOUN']
verb_list = [wp['word'] for wp in word_property if wp['property'] == 'VERB']
print('Input sentence: %s' % input_sentence)
print('Noun list: %s' % ', '.join(noun_list))
print('Verb list: %s\n' % ', '.join(verb_list))
input_sentence = input('Please input a sentence: ')
def check_en_word_similarity():
# 为了使比较算法简洁和快速，spaCy的小模型(所有以sm结尾的包)都不使用单词向量，
# 而且这些sm包只包含上下文相关的向量，我们虽然可以使用similarity()方法，但结果不会太理想
# 所以为了使用真正的词向量，最好使用en_core_web_lg模型
nlp = spacy.load('en_core_web_lg')
tokens = nlp('dog cat banana ssafsf') # ssafsf会被警告改单词没有向量
# 内置单词向量的模型使它们成为可用的标记,文本向量将默认为它们的token向量的平均值,
# 通过has_vector我们可以检查一个token是否有分配的向量，并得到L2规范，它可以用来使向量标准化
print('Token属性：\n')
for token in tokens:
print(token.text, token.has_vector, token.vector_norm, token.is_oov)
print('\nToken两两之间关系：\n')
for token1 in tokens:
for token2 in tokens:
print('%s and %s: %s' % (token1, token2, token1.similarity(token2)))
def check_en_doc_similarity():
nlp = spacy.load('en_core_web_lg')
doc1 = nlp('How to create ticket?')
doc2 = nlp('How can I create ticket?')
doc3 = nlp('I want to create ticket?')
doc4 = nlp('Do you know how to create the ticket?')
print(doc1.similarity(doc2)) # 0.9540794124760449
print(doc2.similarity(doc3)) # 0.9751696134778753
print(doc1.similarity(doc3)) # 0.9549075548119442
print(doc1.similarity(doc4)) # 0.9547450107860063
sentence3 = nlp('Do you know I love you?')
sentence4 = nlp('I love you so much?')
print(sentence3.similarity(sentence4)) # 0.9629862471369796
def check_zh_doc_similarity():
nlp = spacy.load('zh_core_web_lg')
doc1 = nlp('你好吗?')
doc2 = nlp('你还好吗?')
doc3 = nlp('今天你还好吗?')
doc4 = nlp('你的身体今天还好吗?')
print(doc1.similarity(doc2)) # 0.7544851165307768
print(doc2.similarity(doc3)) # 0.9664107589955437
print(doc1.similarity(doc3)) # 0.730822854943996
print(doc1.similarity(doc4)) # 0.6528684500574182
if __name__ == '__main__':
check_word_property()
check_en_word_similarity()
check_zh_doc_similarity()
check_en_doc_similarity()
print('end!')