python文本数据相似度的度量

from nltk.metrics.distance import edit_distancestr1 = 'bad'str2 = 'dad'print(edit_distance(str1, str2))

import nltk#这里展示2元语法text1 = 'Chief Executive Officer'#bigram考虑匹配开头和结束，所有使用pad_right和pad_leftceo_bigrams = nltk.bigrams(text1.split(),pad_right=True,pad_left=True)print(list(ceo_bigrams))[(None, 'Chief'), ('Chief', 'Executive'), ('Executive', 'Officer'), ('Officer', None)]

import nltk#这里展示2元语法def bigram_distance(text1, text2):  #bigram考虑匹配开头和结束，所以使用pad_right和pad_left  text1_bigrams = nltk.bigrams(text1.split(),pad_right=True,pad_left=True)    text2_bigrams = nltk.bigrams(text2.split(), pad_right=True, pad_left=True)    #交集的长度  distance = len(set(text1_bigrams).intersection(set(text2_bigrams)))    return distancetext1 = 'Chief Executive Officer is manager'text2 = 'Chief Technology Officer is technology manager'print(bigram_distance(text1, text2)) #相似度为3

from nltk.metrics.distance import jaccard_distance#这里我们以单个的字符代表文本set1 = set(['a','b','c','d','a'])set2 = set(['a','b','e','g','a'])print(jaccard_distance(set1, set2))

from nltk.metrics.distance import jaccard_distance,masi_distance#这里我们以单个的字符代表文本set1 = set(['a','b','c','d','a'])set2 = set(['a','b','e','g','a'])print(jaccard_distance(set1, set2))print(masi_distance(set1, set2))

word_space = [w1,w2,w3,w4]text1 = 'w1 w2 w1 w4 w1'text2 = 'w1 w3 w2'#按照word_space位置，计算每个位置词语出现的次数text1_vector = [3,1,0,1]text2_vector = [1,1,1,0]

from nltk.cluster.util import cosine_distancetext1_vector = [3,1,0,1]text2_vector = [1,1,1,0]print(cosine_distance(text1_vector,text2_vector))