查询hdfs的数据量

import configparser
import os
import pyhdfs
from queue import Queue
from threading import Threadcategory_queue &＃61; Queue()
hdfs_client &＃61; pyhdfs.HdfsClient(hdfs_ip, hdfs_port, hdfs_user)def load_conf():print("加载配置文件")cf &＃61; configparser.ConfigParser()print(&＃39;---&＃39;)cf.read("base.conf", encoding&＃61;"UTF-8")print(&＃39;&＃61;&＃61;&＃61;&＃61;&＃61;&＃39;)secs &＃61; cf.sections()if len(secs) &＃61;&＃61; 0:raise TypeError("配置文件无效")print("配置文件已加载")print("解析配置文件")conf_dict_list &＃61; [dict()]for x in range(len(conf_dict_list)):sec &＃61; secs[x]ops_list &＃61; cf.options(sec)for ops in ops_list:conf_dict_list[x][ops] &＃61; cf.get(sec, ops)print("配置文件解析成功")return conf_dict_list[0]def get_file():for category_dir in hdfs_client.listdir(hdfs_data_path):num &＃61; 0 # 单类计数if hdfs_client.get_file_status(hdfs_data_path&＃43;os.sep&＃43;category_dir) &＃61;&＃61; &＃39;DIRECTORY&＃39;:category_queue.put(hdfs_data_path&＃43;os.sep&＃43;category_dir)def read_file():while category_queue.not_empty:category_dir &＃61; category_queue.get()num &＃61; 0for date_file in hdfs_client.listdir(category_dir):print(date_file)if &＃39;.out&＃39; in date_file:f &＃61; hdfs_client.open(date_file)line_num &＃61; len(f.readlines())num &＃43;&＃61; line_numprint(category_dir, num)print(&＃39;类别&＃xff1a;{},数据量&＃xff1a;{}&＃39;.format(category_dir, num))if __name__ &＃61;&＃61; &＃39;__main__&＃39;:print("启动程序")base_dict &＃61; load_conf()hdfs_ip &＃61; base_dict["hdfs_ip"]hdfs_port &＃61; int(base_dict["hdfs_port"])hdfs_user &＃61; base_dict["hdfs_user"]hdfs_data_path &＃61; base_dict["hdfs_data_path"]sleep_time &＃61; int(base_dict["sleep_time"])thread_list &＃61; []t &＃61; Thread(target&＃61;get_file)thread_list.append(t)for i in range(10):t &＃61; Thread(target&＃61;read_file)thread_list.append(i)for j in thread_list:j.start()j.join()