vocab文本_持续更新：那些Keras中文本预处理超好用API

import tensorflow as tf
from tensorflow import keras
import numpy as np

from tensorflow.keras.preprocessing.text import Tokenizer# 导入文本数据
with open("shakespeare.txt",&＃39;r&＃39;,encoding&＃61;&＃39;utf&＃61;8&＃39;) as f:text &＃61; f.read()print(text[:100])First Citizen:
Before we proceed any further, hear me speak.All:
Speak, speak.First Citizen:
You# 初始化 tokenizer,oov_token是指语料库中不存在的单词&＃xff0c;这里假定所有不存在的单词都为
tokernizer &＃61; Tokenizer(char_level&＃61;False,oov_token&＃61;&＃39;&＃39;)# 在给定的语料库中训练&＃xff0c;之后tokernizer能够映射任意给定的文本到序列
tokernizer.fit_on_texts([text])sequences &＃61; tokernizer.texts_to_sequences(["Before we proceed any further, hear me speak."])
print(sequences)sequences &＃61; tokernizer.texts_to_sequences(["First Citizen:"])
print(sequences)sequences &＃61; tokernizer.texts_to_sequences(["Hello world and hi"])
print(sequences)[[140, 36, 970, 144, 669, 128, 16, 103]]
[[89, 270]]
[[1, 187, 3, 1]]# tokenizer.word_index 用来查看 tokenizer 中 token被编码的序号
tokenizer.word_index

from tensorflow.keras.preprocessing.text import one_hot# 参数 n 是vocab_size&＃xff0c;字典大小&＃xff0c;应当尽可能大&＃xff0c;否则会出现两个单词相同映射的情况
one_hot("Before we proceed any further ha",n&＃61;128)
[4, 106, 102, 87, 62, 96]# 这里出现了 we 和 any 映射出的整数相同。
one_hot("Before we proceed any further ha",n&＃61;10)
[8, 9, 2, 9, 2, 8]

from tensorflow.keras.preprocessing.text import text_to_word_sequenceprint(text_to_word_sequence("Before we proceed any further, hear me speak.First Citizen:You"))[&＃39;before&＃39;, &＃39;we&＃39;, &＃39;proceed&＃39;, &＃39;any&＃39;, &＃39;further&＃39;, &＃39;hear&＃39;, &＃39;me&＃39;, &＃39;speak&＃39;, &＃39;first&＃39;, &＃39;citizen&＃39;, &＃39;you&＃39;]

&＃39;&＃39;&＃39;
tf.keras.preprocessing.sequence.pad_sequences(sequences, maxlen&＃61;None, dtype&＃61;&＃39;int32&＃39;, padding&＃61;&＃39;pre&＃39;,truncating&＃61;&＃39;pre&＃39;, value&＃61;0.0
)
&＃39;&＃39;&＃39;
t1 &＃61; tf.constant([[1,2,3,7,8],[1,2,5,7,8]])
t1
array([[1, 2, 3, 7, 8],[1, 2, 5, 7, 8]])># 默认 padding&＃61;&＃39;pre&＃39;&＃xff0c;在前方填充&＃xff0c;默认value&＃61;0.0&＃xff0c;使用0值填充
keras.preprocessing.sequence.pad_sequences(t1,maxlen&＃61;7) # 将t1 填充至(2,7)
array([[0, 0, 1, 2, 3, 7, 8],[0, 0, 1, 2, 5, 7, 8]])# 指定 padding&＃61;&＃39;post&＃39;&＃xff0c;在后方填充
keras.preprocessing.sequence.pad_sequences(t1,maxlen&＃61;7,padding&＃61;&＃39;post&＃39;)
array([[1, 2, 3, 7, 8, 0, 0],[1, 2, 5, 7, 8, 0, 0]])# 使用指定值填充
keras.preprocessing.sequence.pad_sequences(t1,maxlen&＃61;7,padding&＃61;&＃39;post&＃39;,value&＃61;1)
array([[1, 2, 3, 7, 8, 1, 1],[1, 2, 5, 7, 8, 1, 1]])# truncating&＃61;&＃39;pre&＃39; 当序列长度超过maxlen参数&＃xff0c;会从前方截断&＃xff0c;post则从后方截断
keras.preprocessing.sequence.pad_sequences(t1,maxlen&＃61;3,truncating&＃61;&＃39;post&＃39;)
array([[1, 2, 3],[1, 2, 5]])

from tensorflow.keras.utils import to_categoricaly &＃61; [0,0,3,5,1,4,2,2,1]
to_categorical(y)
array([[1., 0., 0., 0., 0., 0.],[1., 0., 0., 0., 0., 0.],[0., 0., 0., 1., 0., 0.],[0., 0., 0., 0., 0., 1.],[0., 1., 0., 0., 0., 0.],[0., 0., 0., 0., 1., 0.],[0., 0., 1., 0., 0., 0.],[0., 0., 1., 0., 0., 0.],[0., 1., 0., 0., 0., 0.]], dtype&＃61;float32)# 指定 class 的数量
y &＃61; [0,0,3,5,1,4,2,2,1]
to_categorical(y,num_classes&＃61;8)
array([[1., 0., 0., 0., 0., 0., 0., 0.],[1., 0., 0., 0., 0., 0., 0., 0.],[0., 0., 0., 1., 0., 0., 0., 0.],[0., 0., 0., 0., 0., 1., 0., 0.],[0., 1., 0., 0., 0., 0., 0., 0.],[0., 0., 0., 0., 1., 0., 0., 0.],[0., 0., 1., 0., 0., 0., 0., 0.],[0., 0., 1., 0., 0., 0., 0., 0.],[0., 1., 0., 0., 0., 0., 0., 0.]], dtype&＃61;float32)

vocab &＃61; ["Up",&＃39;Down&＃39;,&＃39;Left&＃39;,&＃39;Right&＃39;] # 词汇表&＃xff0c;所有可能的类别的列表
indices &＃61; tf.range(len(vocab),dtype&＃61;tf.int64) # 词汇表索引的张量
table_init &＃61; tf.lookup.KeyValueTensorInitializer(vocab,indices)
num_oov &＃61; 3
table &＃61; tf.lookup.StaticVocabularyTable(table_init,num_oov) # 查找表
# 创建了查找表之后&＃xff0c;我们就可以对任意的类别进行索引编码
categories &＃61; tf.constant([&＃39;Down&＃39;,&＃39;Down&＃39;,&＃39;Right&＃39;,&＃39;Up&＃39;,&＃39;Up&＃39;,&＃39;Unknown&＃39;])
table.lookup(categories)