python中文自然语言处理_Python中文自然语言处理：一、基础文本处理

对中文进行分词

import jieba

text &＃61; &＃39;你好&＃xff0c;我正在进行Python自然语言处理&＃xff0c;有些问题需要处理,笑哈哈&＃39;

word &＃61; jieba.cut(text)

word_list &＃61; &＃39; &＃39;.join(word).split(&＃39; &＃39;)

print(word_list)

输出&＃xff1a;

[&＃39;你好&＃39;, &＃39;&＃xff0c;&＃39;, &＃39;我&＃39;, &＃39;正在&＃39;, &＃39;进行&＃39;, &＃39;Python&＃39;, &＃39;自然语言&＃39;, &＃39;处理&＃39;, &＃39;&＃xff0c;&＃39;, &＃39;有些&＃39;, &＃39;问题&＃39;, &＃39;需要&＃39;, &＃39;处理&＃39;, &＃39;,&＃39;, &＃39;笑哈哈&＃39;]

去除文本中的标点符号

import re

reg &＃61; r"[\s&＃43;\.\!\/_,$%^*(&＃43;\"\&＃39;]&＃43;|[&＃43;——&＃xff01;&＃xff0c;。&＃xff1f;、~&＃64;#&＃xffe5;%……&*&＃xff08;&＃xff09;]&＃43;"

for i in word_list:

result &＃61; re.match(reg,i)

if result !&＃61; None:

word_list.remove(i)

print(word_list)

输出&＃xff1a;

[&＃39;你好&＃39;, &＃39;我&＃39;, &＃39;正在&＃39;, &＃39;进行&＃39;, &＃39;Python&＃39;, &＃39;自然语言&＃39;, &＃39;处理&＃39;, &＃39;有些&＃39;, &＃39;问题&＃39;, &＃39;需要&＃39;, &＃39;处理&＃39;, &＃39;笑哈哈&＃39;]

生成一个单词的起始位置

text_no_punp &＃61; re.sub("[\s&＃43;\.\!\/_,$%^*(&＃43;\"\&＃39;]&＃43;|[&＃43;——&＃xff01;&＃xff0c;。&＃xff1f;、~&＃64;#&＃xffe5;%……&*&＃xff08;&＃xff09;]&＃43;", "",text)

print(list(jieba.tokenize(text_no_punp)))

输出&＃xff1a;

[(&＃39;你好&＃39;, 0, 2), (&＃39;我&＃39;, 2, 3), (&＃39;正在&＃39;, 3, 5), (&＃39;进行&＃39;, 5, 7), (&＃39;Python&＃39;, 7, 13), (&＃39;自然语言&＃39;, 13, 17), (&＃39;处理&＃39;, 17, 19), (&＃39;有些&＃39;, 19, 21), (&＃39;问题&＃39;, 21, 23), (&＃39;需要&＃39;, 23, 25), (&＃39;处理&＃39;, 25, 27), (&＃39;笑哈哈&＃39;, 27, 30)]

去除重复词

class RepeatReplacer(object):

def __init__(self):

self.repeat_regexp &＃61; re.compile(r&＃39;(\w*)(\w)\2(\w*)&＃39;)

self.repl &＃61; r&＃39;\1\2\3&＃39;

def replace(self,word):

repl_word &＃61; self.repeat_regexp.sub(self.repl,word)

if repl_word !&＃61; word:

return self.replace(repl_word)

else:

return repl_word

replacer &＃61; RepeatReplacer()

replacer.replace("高高兴兴")

输出&＃xff1a;

&＃39;高兴&＃39;

对文本应用Zipf定律

import nltk

from nltk.corpus import brown

from nltk.probability import FreqDist

import matplotlib

import matplotlib.pyplot as plt

# 解决中文和负号显示

from pylab import mpl

mpl.rcParams[&＃39;font.sans-serif&＃39;] &＃61; [&＃39;SimHei&＃39;]

mpl.rcParams[&＃39;axes.unicode_minus&＃39;] &＃61; False

matplotlib.use(&＃39;MacOSX&＃39;)

fd &＃61; FreqDist()

for text in gutenberg.fileids():

for word in gutenberg.words(text):

fd[word]&＃43;&＃61; 1

ranks &＃61; []

freqs &＃61; []

for rank, word in enumerate(fd):

ranks.append(rank&＃43;1)

freqs.append(fd[word])

plt.figure(figsize&＃61;(15,8))

plt.loglog(ranks,freqs,&＃39;.-&＃39;)

plt.xlabel(&＃39;词频(f)&＃39;, fontsize&＃61;14, fontweight&＃61;&＃39;bold&＃39;)

plt.ylabel(&＃39;排名(r)&＃39;, fontsize&＃61;14, fontweight&＃61;&＃39;bold&＃39;)

plt.grid(True)

plt.show()

相似性度量

from nltk.metrics import *

text1 &＃61; &＃39;你好&＃xff0c;我正在使用Python自然语言处理&＃xff0c;有些问题正在处理,嘿嘿&＃39;

word1 &＃61; jieba.cut(text1)

word_list1 &＃61; &＃39; &＃39;.join(word1).split(&＃39; &＃39;)

# print(word_list1)

for i in word_list1:

result &＃61; re.match(reg,i)

if result !&＃61; None:

word_list1.remove(i)

print(word_list)

print(word_list1)

# 准确性度量

print(accuracy(word_list,word_list1))

#　Jaccard相似系数度量

print(jaccard_distance(set(word_list),set(word_list1)))

# MASI距离度量

print(masi_distance(set(word_list),set(word_list1)))

# 二值距离度量

print(binary_distance(set(word_list),set(word_list1)))

输出&＃xff1a;

[&＃39;你好&＃39;, &＃39;我&＃39;, &＃39;正在&＃39;, &＃39;进行&＃39;, &＃39;Python&＃39;, &＃39;自然语言&＃39;, &＃39;处理&＃39;, &＃39;有些&＃39;, &＃39;问题&＃39;, &＃39;需要&＃39;, &＃39;处理&＃39;, &＃39;笑哈哈&＃39;]

[&＃39;你好&＃39;, &＃39;我&＃39;, &＃39;正在&＃39;, &＃39;使用&＃39;, &＃39;Python&＃39;, &＃39;自然语言&＃39;, &＃39;处理&＃39;, &＃39;有些&＃39;, &＃39;问题&＃39;, &＃39;正在&＃39;, &＃39;处理&＃39;, &＃39;嘿嘿&＃39;]

0.75

0.38461538461538464

0.12692307692307692

1.0

分享到&＃xff1a;