Pandas速查

如果不归一化&＃xff0c;各维特征的跨度差距很大&＃xff0c;目标函数就会是“扁”的&＃xff0c;图中椭圆表示目标函数的等高线&＃xff0c;两个坐标轴代表两个特征。
在进行梯度下降的时候&＃xff0c;梯度的方向就会偏离最小值的方向&＃xff0c;走很多弯路。 归一化后&＃xff0c;那么目标函数就变“圆”了&＃xff0c;每一步梯度的方向都基本指向最小值&＃xff0c;可以大踏步地前进&＃xff0c;每一步梯度的方向基本都指向最小值&＃xff0c;可以大踏步的前进。

a&＃61;"from.US.NY"
print(&＃39;-&＃39;.join(a.split(&＃39;.&＃39;)[:2]))mm_ids &＃61; (123,456,789)
sql &＃61; "id in (%s) " % &＃39;,&＃39;.join([str(x) for x in mm_ids])
print(sql)
from-US
id in (123,456,789) import pandas as pd
import numpy as np
pd.set_option(&＃39;display.max_column&＃39;,20)
pd.set_option(&＃39;display.width&＃39;,200)
np.set_printoptions(suppress&＃61;True)# 列表构造 DataFrame
ll &＃61; []
for i in np.arange(29):ll.append(result[result["Y"] &＃61;&＃61; i].mean().values)
df_y_mean &＃61; pd.DataFrame(ll, columns&＃61;result.columns).T# 索引过滤
df_y_mean &＃61; df_y_mean[df_y_mean.index!&＃61;&＃39;Y&＃39;]# 像素
plt.rcParams[&＃39;figure.dpi&＃39;] &＃61; 250
df_y_mean.plot(kind&＃61;&＃39;bar&＃39;, x&＃61;&＃39;index&＃39;, y&＃61;np.arange(29), subplots&＃61;True, figsize&＃61;(50, 320), grid&＃61;True, title&＃61;"Event_Distribution",rot&＃61;90, fontsize&＃61;2)# 过滤空值 和 零值
df &＃61; df.dropna(subset&＃61;[&＃39;lat&＃39;,&＃39;lng&＃39;], axis&＃61;0, how&＃61;&＃39;any&＃39;)
df &＃61; df[(df.lat!&＃61;0)&(df.lng!&＃61;0)]# 分组 标准差 分散值 线形图
grouped &＃61; df.groupby("mm_id",as_index&＃61;True)
grouped_gps &＃61; grouped[[&＃39;mm_id&＃39;,&＃39;lat&＃39;,&＃39;lng&＃39;]]
grouped_gps_std &＃61; grouped_gps.agg(&＃39;std&＃39;)
grouped_gps_std[&＃39;disperse&＃39;] &＃61; grouped_gps_std[&＃39;lat&＃39;] &＃43; grouped_gps_std[&＃39;lng&＃39;]
plt.figure()
grouped_gps_std.sort_values(by&＃61;&＃39;disperse&＃39;,ascending&＃61;True).set_index(np.arange(0,len(grouped_gps_std))).loc[:,&＃39;disperse&＃39;].plot(subplots&＃61;True,figsize&＃61;(10,5),grid&＃61;True,title&＃61;"mm_id_gps_disperse")
plt.savefig("id_gps_disperse.png",dpi &＃61; 1000)# 快速构建 DataFrame
df&＃61;pd.DataFrame({&＃39;A&＃39;:[&＃39;foo&＃39;,&＃39;bar&＃39;,&＃39;foo&＃39;,&＃39;bar&＃39;,&＃39;foo&＃39;,&＃39;bar&＃39;,&＃39;yyy&＃39;,&＃39;xxx&＃39;],&＃39;B&＃39;:[&＃39;one&＃39;,&＃39;one&＃39;,&＃39;two&＃39;,&＃39;three&＃39;,&＃39;two&＃39;,&＃39;two&＃39;,&＃39;one&＃39;,&＃39;three&＃39;],&＃39;C&＃39;:np.random.randn(8),&＃39;D&＃39;:np.random.randn(8)})

# 将字符串类型的 dict list tuple 转化成原型
df_origin[&＃39;action_num_map_dict&＃39;] &＃61; df_origin[&＃39;action_num_map&＃39;].map(lambda x: ast.literal_eval(x))# 字典 构造 DataFrame
df_event &＃61; pd.DataFrame(list(df_origin.action_num_map_dict)).fillna(value&＃61;0)
df &＃61; pd.concat([df_origin.mm_id,df_event],axis&＃61;1) #默认按index粘连
df &＃61; df.set_index("mm_id",drop&＃61;True)# 列表成员 所在的位置
jb &＃61; [index for index,item in enumerate(df.columns.map(lambda x: x.split(&＃39;.&＃39;)[0]&＃61;&＃61;&＃39;live&＃39;)) if item&＃61;&＃61;True] #包含&＃39;live&＃39;的column所在位置# 筛选特定列
df &＃61; df.loc[:,df.columns.map(lambda x: x.split(&＃39;.&＃39;)[0]&＃61;&＃61;&＃39;live&＃39;)]# 按行求和 按列求和
df &＃61; df[df.sum(1)!&＃61;0]
df &＃61; df.loc[:,df.sum()!&＃61;0]# 黏连
result &＃61; pd.merge(df,pd.DataFrame(labels,columns&＃61;[&＃39;Y&＃39;],index&＃61;df.index),left_index&＃61;True,right_index&＃61;True)
snatch_or_lists &＃61; pd.merge(snatch,lists,on&＃61;&＃39;momo_id&＃39;,how&＃61;&＃39;outer&＃39;,suffixes&＃61;("_snatch","_lists"))# 列 类型转换
df_origin[&＃39;action_num_map&＃39;] &＃61; df_origin[&＃39;action_num_map&＃39;].apply(_to_dict)# if条件 列表推导
abnormal_id &＃61; [item for item in df_snatch[&＃39;mm_id&＃39;] if item not in df_profile[&＃39;mm_id&＃39;]]# 在一个里面不在另一个里面
ids &＃61; snatch[~snatch.mm_id.isin(mm_id_profile)]
ids &＃61; lists[~lists.mm_id.isin(mm_id_profile)]# 两列 联合 求并集
def foo(d):xx &＃61; set(d.roomid_snatch)yy &＃61; set(d.roomid_lists) return len([item for item in xx.union(yy) if item])
hongbao_rooms &＃61; snatch_or_lists.groupby(&＃39;mm_id&＃39;).apply(foo)
hongbao_rooms &＃61; pd.DataFrame(hongbao_rooms,columns&＃61;[&＃39;hb_rooms&＃39;])# 纵向堆叠 分组统计 进入的总房间数量
all &＃61; pd.concat([snatch[[&＃39;mm_id&＃39;,&＃39;roomid&＃39;]],lists[[&＃39;mm_id&＃39;,&＃39;roomid&＃39;]],shared[[&＃39;mm_id&＃39;,&＃39;roomid&＃39;]],profile[[&＃39;mm_id&＃39;,&＃39;roomid&＃39;]]],ignore_index&＃61;True).groupby(&＃39;mm_id&＃39;).nunique()# 列 重命名
all &＃61; all.rename(columns&＃61;{"roomid": "all_nums"})# 分组统计 序列 转 dataframe
df &＃61; DataFrame({&＃39;key1&＃39;:[&＃39;a&＃39;,&＃39;a&＃39;,&＃39;b&＃39;,&＃39;b&＃39;,&＃39;a&＃39;],&＃39;key2&＃39;:[&＃39;one&＃39;,&＃39;one&＃39;,&＃39;one&＃39;,&＃39;two&＃39;,&＃39;one&＃39;]}
)
print(df)
df[&＃39;key2&＃39;].groupby(df[&＃39;key1&＃39;]).nunique().to_frame() key1 key2
0 a one
1 a one
2 b one
3 b two
4 a one
Out[20]:
key2
key1
a 1
b 2# 分组 去重统计 列命名
exit_rooms &＃61; exit[&＃39;roomid&＃39;].groupby(exit[&＃39;mm_id&＃39;]).nunique().to_frame().rename(columns&＃61;{&＃39;roomid&＃39;:&＃39;exit_nums&＃39;})# 分组 按时间排序
df &＃61; snatch.groupby([&＃39;mm_id&＃39;],sort&＃61;False).apply(lambda df: df.sort_values(by&＃61;&＃39;access_timestamp&＃39;, ascending&＃61;True))# 保留首次出现
df_1st &＃61; df.drop_duplicates(["mm_id", "roomid"], keep&＃61;&＃39;first&＃39;)# 字符串时间 标准格式时间
start_date &＃61; datetime.strptime(end_date, &＃39;%Y%m%d&＃39;) - timedelta(days&＃61;30)
start_date &＃61; datetime.strftime(start_date, &＃39;%Y%m%d&＃39;)# 时间戳 转 字符串日期
df_month[&＃39;day&＃39;] &＃61; df_month[&＃39;timestamp&＃39;].apply(lambda x: datetime.fromtimestamp(x).strftime(&＃39;%Y%m%d&＃39;))# 循环读取文件 黏连
pieces &＃61; []
columns &＃61; [&＃39;mm_id&＃39;,&＃39;timestamp&＃39;,&＃39;types&＃39;]
for days in range_days(start_date, end_date):path &＃61; &＃39;/name1/name2/information_scan_%s.csv&＃39; %daysframe &＃61; pd.read_csv(path)frame[&＃39;day&＃39;] &＃61; dayspieces.append(frame)
df &＃61; pd.concat(pieces,ignore_index&＃61;True)

# 均匀切分 分组统计
df_month &＃61; pd.DataFrame({&＃39;days&＃39;: random.sample(range(0,30),10)})[&＃39;days&＃39;] # 序列
factor &＃61; pd.cut(df_month,3)
def get_stats(group):return {&＃39;min&＃39;:group.min(),&＃39;max&＃39;:group.max(),&＃39;count&＃39;:group.count(),&＃39;mean&＃39;:group.mean()}
grouped &＃61; df_month.groupby(factor)
print(grouped.apply(get_stats))
grouped.apply(get_stats).unstack()
days
(1.974, 10.667] count 3.000000max 5.000000mean 3.666667min 2.000000
(10.667, 19.333] count 4.000000max 17.000000mean 14.750000min 12.000000
(19.333, 28.0] count 3.000000max 28.000000mean 26.000000min 23.000000
Name: days, dtype: float64count max mean min
days
(1.974, 10.667] 3.0 5.0 3.666667 2.0
(10.667, 19.333] 4.0 17.0 14.750000 12.0
(19.333, 28.0] 3.0 28.0 26.000000 23.0# 透视表
df_day &＃61; pd.pivot_table(df_day,index&＃61;&＃39;mm_id&＃39;,columns&＃61;&＃39;day&＃39;,aggfunc&＃61;len)# 分位点 统计
df_day.describe(percentiles&＃61;[0.25,0.75,0.85,0.90,0.95,0.99])# 字典
feature[&＃39;event_summary&＃39;] &＃61; feature[&＃39;event_summary&＃39;].apply(lambda x: {k: int(v) for k, v in x.items()})# Hash Trick 降维
h &＃61; FeatureHasher(n_features&＃61;50)
f &＃61; h.transform(feature[&＃39;event_summary&＃39;].values)
df &＃61; pd.DataFrame(f.toarray(), index&＃61;feature.index, columns&＃61;np.arange(0, 50))
feature &＃61; pd.concat([feature, df], axis&＃61;1)# 缺失值 填补
imp &＃61; Imputer(missing_values&＃61;&＃39;NaN&＃39;, strategy&＃61;&＃39;most_frequent&＃39;, axis&＃61;0)
imp.fit(feature[&＃39;country&＃39;].values.reshape(-1,1))
feature.loc[:, &＃39;country&＃39;] &＃61; imp.transform(feature[&＃39;country&＃39;].values.reshape(-1, 1))# 独热码
df &＃61; pd.get_dummies(feature[&＃39;gender&＃39;], prefix&＃61;&＃39;gender_&＃39;)# 二值化
binarizer &＃61; preprocessing.Binarizer(threshold&＃61;10)
feature[&＃39;version&＃39;] &＃61; binarizer.transform(feature.version.values.reshape(-1,1))# 随机过采样
ros &＃61; RandomOverSampler(random_state&＃61;9)
X_train, y_train &＃61; ros.fit_sample(X_train, y_train)# 评估
precision &＃61; precision_score(y.values, y_pred, average&＃61;&＃39;binary&＃39;)
recall &＃61; recall_score(y.values, y_pred, average&＃61;&＃39;binary&＃39;)# 二值化
trainset[&＃39;labels&＃39;] &＃61; trainset.SALE_ACTION_y.map(lambda x: np.sign(x))# 加权投票
lr &＃61; LogisticRegression(class_weight&＃61;&＃39;balanced&＃39;)
rf &＃61; RandomForestClassifier(oob_score&＃61;True, random_state&＃61;9)
gbm &＃61; GradientBoostingClassifier(random_state&＃61;9)
eclf3 &＃61; VotingClassifier(estimators&＃61;[(&＃39;logistic regression&＃39;, lr), (&＃39;randomForest classifier&＃39;, rf), (&＃39;gbdt&＃39;, gbm)],voting &＃61; &＃39;soft&＃39;, weights &＃61; [20, 95, 40])
eclf3 &＃61; eclf3.fit(X_train, y_train)
y_pred &＃61; eclf3.predict(X_test.values)# 日期
feature &＃61; pd.DataFrame({&＃39;register_time&＃39;: [&＃39;20181212&＃39;,&＃39;20181112&＃39;]})
feature[&＃39;register_time&＃39;] &＃61; pd.to_datetime(feature[&＃39;register_time&＃39;], errors&＃61;&＃39;coerce&＃39;)
feature[&＃39;days&＃39;] &＃61; datetime.now() - feature[&＃39;register_time&＃39;]
feature[&＃39;days&＃39;] &＃61; feature[&＃39;days&＃39;].map(lambda x: x.days)
feature
Out[38]:
register_time days
0 2018-12-12 30
1 2018-11-12 60events &＃61; df.columns.map(lambda x: &＃39;.&＃39;.join(x.split(&＃39;.&＃39;)[:2]))
df.columns &＃61; eventscolumn_once&＃61;[]
column_all&＃61;list(df.columns)
for i in column_all:if column_all.count(i)&＃61;&＃61;1:column_once.append(i)df_tmp &＃61; pd.DataFrame(index&＃61;df.index)
for item in set(df.columns):if item not in column_once:df_tmp&＃61;pd.concat([df_tmp,df[item].sum(1).to_frame(item)], axis&＃61;1)else:df_tmp&＃61;pd.concat([df_tmp, df[item]], axis&＃61;1)# Redis 写入
REDIS_HOST &＃61; &＃39;200.0.0.1&＃39;
REDIS_PORT &＃61; 4321
key &＃61; &＃39;test1&＃39;
client &＃61; redis.Redis(host&＃61;REDIS_HOST, port&＃61;REDIS_PORT)def update_redis(conn, key, data_set):origin_data &＃61; set()for member in conn.sscan_iter(key):origin_data.add(member)for member in (origin_data - data_set):conn.srem(key, member)time.sleep(0.01)for member in (data_set - origin_data):conn.sadd(key, member)time.sleep(0.01)# 分组 去重统计 转化
df &＃61; df.groupby(&＃39;mm_id&＃39;)[&＃39;day&＃39;].nunique().to_frame().rename(columns&＃61;{&＃39;day&＃39;: &＃39;days&＃39;})# 个数
df &＃61; df.groupby([&＃39;mm_id&＃39;, &＃39;day&＃39;]).size()# 迭代器
df &＃61; pd.DataFrame(columns&＃61;[&＃39;mm_id&＃39;,&＃39;timestamp&＃39;,&＃39;type&＃39;])
this, last &＃61; None, None
for index, row in df_origin_sort.iterrows():last &＃61; thisthis &＃61; rowif last is None:continueif (this.mm_id&＃61;&＃61;last.mm_id) & (this.type&＃61;&＃61;&＃39;downline&＃39;) & (last.type&＃61;&＃61;&＃39;online&＃39;) & ((this.timestamp-last.timestamp) >&＃61; 4*3600):df &＃61; df.append([last,this],ignore_index&＃61;True)