python中有matlab中的lwt吗_在Python中简单实现Ngram,tfidf和余弦相似度

检查NLTK包&＃xff1a;

http://www.nltk.org它有一切你需要的

对于余弦相似性&＃xff1a;

def cosine_distance(u, v):

"""

Returns the cosine of the angle between vectors v and u. This is equal to

u.v / |u||v|.

"""

return numpy.dot(u, v) / (math.sqrt(numpy.dot(u, u)) * math.sqrt(numpy.dot(v, v)))

对于ngram&＃xff1a;

def ngrams(sequence, n, pad_left&＃61;False, pad_right&＃61;False, pad_symbol&＃61;None):

"""

A utility that produces a sequence of ngrams from a sequence of items.

For example:

>>> ngrams([1,2,3,4,5], 3)

[(1, 2, 3), (2, 3, 4), (3, 4, 5)]

Use ingram for an iterator version of this function. Set pad_left

or pad_right to true in order to get additional ngrams:

>>> ngrams([1,2,3,4,5], 2, pad_right&＃61;True)

[(1, 2), (2, 3), (3, 4), (4, 5), (5, None)]

&＃64;param sequence: the source data to be converted into ngrams

&＃64;type sequence: C{sequence} or C{iterator}

&＃64;param n: the degree of the ngrams

&＃64;type n: C{int}

&＃64;param pad_left: whether the ngrams should be left-padded

&＃64;type pad_left: C{boolean}

&＃64;param pad_right: whether the ngrams should be right-padded

&＃64;type pad_right: C{boolean}

&＃64;param pad_symbol: the symbol to use for padding (default is None)

&＃64;type pad_symbol: C{any}

&＃64;return: The ngrams

&＃64;rtype: C{list} of C{tuple}s

"""

if pad_left:

sequence &＃61; chain((pad_symbol,) * (n-1), sequence)

if pad_right:

sequence &＃61; chain(sequence, (pad_symbol,) * (n-1))

sequence &＃61; list(sequence)

count &＃61; max(0, len(sequence) - n &＃43; 1)

return [tuple(sequence[i:i&＃43;n]) for i in range(count)]

对于tf-idf&＃xff0c;你将不得不首先计算分布&＃xff0c;我使用Lucene做到这一点&＃xff0c;但你可能很好地做类似于NLTK的东西&＃xff0c;使用FreqDist&＃xff1a;

如果你喜欢pylucene&＃xff0c;这将告诉你如何喜欢tf.idf

# reader &＃61; lucene.IndexReader(FSDirectory.open(index_loc))

docs &＃61; reader.numDocs()

for i in xrange(docs):

tfv &＃61; reader.getTermFreqVector(i, fieldname)

if tfv:

rec &＃61; {}

terms &＃61; tfv.getTerms()

frequencies &＃61; tfv.getTermFrequencies()

for (t,f,x) in zip(terms,frequencies,xrange(maxtokensperdoc)):

df&＃61; searcher.docFreq(Term(fieldname, t)) # number of docs with the given term

tmap.setdefault(t, len(tmap))

rec[t] &＃61; sim.tf(f) * sim.idf(df, max_doc) #compute TF.IDF

# and normalize the values using cosine normalization

if cosine_normalization:

denom &＃61; sum([x**2 for x in rec.values()])**0.5

for k,v in rec.items():

rec[k] &＃61; v / denom