PyTorch制作图片数据集

PyTorch制作图片数据集

图片的处理

os.listdir()函数

>>> path &＃61; &＃39;./data/training_set/training_set/cats&＃39;
>>> os.listdir(path)
[&＃39;cat.1.jpg&＃39;, &＃39;cat.10.jpg&＃39;, &＃39;cat.100.jpg&＃39;, &＃39;cat.1000.jpg&＃39;, &＃39;cat.1001.jpg&＃39;, &＃39;cat.1002.jpg&＃39;, &＃39;cat.1003.jpg&＃39;, &＃39;cat.1004.jpg&＃39;, &＃39;cat.1005.jpg&＃39;, &＃39;cat.1006.jpg&＃39;, &＃39;cat.1007.jpg&＃39;, &＃39;cat.1008.jpg&＃39;, &＃39;cat.1009.jpg&＃39;, &＃39;cat.101.jpg&＃39;, &＃39;cat.1010.jpg&＃39;, &＃39;cat.1011.jpg&＃39;, &＃39;cat.1012.jpg&＃39;, &＃39;cat.1013.jpg&＃39;, &＃39;cat.1014.jpg&＃39;, &＃39;cat.1015.jpg&＃39;, &＃39;
cat.1016.jpg&＃39;, &＃39;cat.1017.jpg&＃39;, &＃39;cat.1018.jpg&＃39;, &＃39;cat.1019.jpg&＃39;, &＃39;cat.102.jpg&＃39;, &＃39;cat.1020.jpg&＃39;, &＃39;cat.1021.jpg&＃39;, &＃39;cat.1022.jpg&＃39;, &＃39;cat.1023.jpg&＃39;, &＃39;cat.1024.jpg&＃39;, &＃39;cat.1025.jpg&＃39;, &＃39;cat.1026.jpg&＃39;, &＃39;cat.1027.jpg&＃39;, &＃39;cat.1028.jpg&＃39;, &＃39;cat.1029.jpg&＃39;, &＃39;cat.103.jpg&＃39;, &＃39;cat.1030.jpg&＃39;, &＃39;cat.1031.jpg&＃39;, &＃39;cat.1032.jpg&＃39;, &＃39;cat.1033.jpg&＃39;
, &＃39;cat.1034.jpg&＃39;, &＃39;cat.1035.jpg&＃39;, &＃39;cat.1036.jpg&＃39;, &＃39;cat.1037.jpg&＃39;, &＃39;cat.1038.jpg&＃39;, &＃39;cat.1039.jpg&＃39;, &＃39;cat.104.jpg&＃39;, &＃39;cat.1040.jpg&＃39;, &＃39;cat.1041.jpg&＃39;, &＃39;cat.1042.jpg&＃39;, &＃39;cat.1043.jpg&＃39;, &＃39;cat.1044.jpg&＃39;, &＃39;cat.1045.jpg&＃39;, &＃39;cat.1046.jpg&＃39;, &＃39;cat.1047.jpg&＃39;, &＃39;cat.1048.jpg&＃39;, &＃39;cat.1049.jpg&＃39;, &＃39;cat.105.jpg&＃39;, &＃39;cat.1050.jpg&＃39;, &＃39;cat.1051.j
pg&＃39;, &＃39;cat.1052.jpg&＃39;, &＃39;cat.1053.jpg&＃39;, &＃39;cat.1054.jpg&＃39;, &＃39;cat.1055.jpg&＃39;, &＃39;cat.1056.jpg&＃39;, &＃39;cat.1057.jpg&＃39;, &＃39;cat.1058.jpg&＃39;, &＃39;cat.1059.jpg&＃39;, &＃39;cat.106.jpg&＃39;, &＃39;cat.1060.jpg&＃39;, &＃39;cat.1061.jpg&＃39;, &＃39;cat.1062.jpg&＃39;, &＃39;cat.1063.jpg&＃39;, &＃39;cat.1064.jpg&＃39;, &＃39;cat.1065.jpg&＃39;, &＃39;cat.1066.jpg&＃39;, &＃39;cat.1067.jpg&＃39;, &＃39;cat.1068.jpg&＃39;, &＃39;cat.1069.jpg&＃39;, &＃39;cat.10
7.jpg&＃39;, &＃39;cat.1070.jpg&＃39;, &＃39;cat.1071.jpg&＃39;, &＃39;cat.1072.jpg&＃39;, &＃39;cat.1073.jpg&＃39;, &＃39;cat.1074.jpg&＃39;, &＃39;cat.1075.jpg&＃39;, &＃39;cat.1076.jpg&＃39;, &＃39;cat.1077.jpg&＃39;, &＃39;cat.1078.jpg&＃39;, &＃39;cat.1079.jpg&＃39;, &＃39;cat.108.jpg&＃39;, &＃39;cat.1080.jpg&＃39;, &＃39;cat.1081.jpg&＃39;, &＃39;cat.1082.jpg&＃39;, &＃39;cat.1083.jpg&＃39;, &＃39;cat.1084.jpg&＃39;, &＃39;cat.1085.jpg&＃39;, &＃39;cat.1086.jpg&＃39;, &＃39;cat.1087.jpg&＃39;, &＃39;cat
.1088.jpg&＃39;, &＃39;cat.1089.jpg&＃39;, &＃39;cat.109.jpg&＃39;, &＃39;cat.1090.jpg&＃39;, &＃39;cat.1091.jpg&＃39;, &＃39;cat.1092.jpg&＃39;, &＃39;cat.1093.jpg&＃39;, &＃39;cat.1094.jpg&＃39;, &＃39;cat.1095.jpg&＃39;, &＃39;cat.1096.jpg&＃39;, &＃39;cat.1097.jpg&＃39;, &＃39;cat.1098.jpg&＃39;, &＃39;cat.1099.jpg&＃39;, &＃39;cat.11.jpg&＃39;, &＃39;cat.110.jpg&＃39;, &＃39;cat.1100.jpg&＃39;, &＃39;cat.1101.jpg&＃39;, &＃39;cat.1102.jpg&＃39;, &＃39;cat.1103.jpg&＃39;, &＃39;cat.1104.jpg&＃39;, &＃39;...]


可以看出作用是返回path路径文件夹下所有文件名的列表

torchvision的transform模块

>>> import torch
>>> from PIL import Image
>>> from torchvision import transforms
>>> img &＃61; Image.open(&＃39;./data/training_set/training_set/cats/cat.1.jpg&＃39;)
>>> img.size
(300, 280)
>>> transforms.Resize(256)(img).size # 比例改变大小
(274, 256)
>>> transforms.Resize([256,256])(img).size # 结果为图-比例缩小
(256, 256)
>>> transforms.RandomResizedCrop(256)(img).size # 随机切为目标大小
(256, 256)
>>> transforms.RandomSizedCrop([256,200])(img).size # 随机切2
(200, 256)
>>> transforms.Pad(20,1)(img).size # 第一个参数为填充大小&＃xff0c;第二个参数为填充值
(340, 320)
>>> transforms.CenterCrop(256)(img).size # 中心切
(256, 256)

原图&＃xff1a;

比例缩小&＃xff1a;

随机切&＃xff1a;

随机切2&＃xff1a;

中心切&＃xff1a;

填充&＃xff1a;

>>> transforms.ToTensor()(img).shape
torch.Size([3, 280, 300])


transforms.ToTensor()函数可以将PIL.Image对象转换为tensor对象
维度由[H, W, C]转为[C, H, W]
H&＃xff1a;Height
W&＃xff1a;Width
C&＃xff1a;Channel
上面的函数一般在ToTensor函数之前进行&＃xff0c;下面的函数对tensor进行操作&＃xff0c;一般在ToTensor之后进行

>>> imgt &＃61; transforms.ToTensor()(img)
>>> imgt.shape
torch.Size([3, 280, 300])
>>> transforms.ToPILImage()(imgt).size # tensor转化为PILImage
(300, 280)
>>> m &＃61; imgt.mean(axis &＃61; [1,2]) # 求均值&＃xff0c;axis&＃61;[1,2]可以看作关闭了后两个维度&＃xff0c;即在后两个维度上求均值
>>> m
tensor([0.3089, 0.2677, 0.2665])
>>> s &＃61; imgt.std(axis &＃61; [1,2]) # 求标准差
>>> s
tensor([0.1619, 0.1375, 0.1380])
>>> imgtn &＃61; transforms.Normalize(m,s)(imgt) # 对tensor标准化
>>> imgtn.shape
torch.Size([3, 280, 300])
>>> imgtnt &＃61; torch.zeros(3,280,300)
>>> imgtnt[0] &＃61; (imgt[0]-m[0])/s[0]
>>> imgtnt[1] &＃61; (imgt[1]-m[1])/s[1]
>>> imgtnt[2] &＃61; (imgt[2]-m[2])/s[2]
>>> imgtnt &＃61;&＃61; imgtn # 标准化过程就是对每个通道output &＃61; (input-mean)/std
tensor([[[True, True, True, ..., True, True, True],[True, True, True, ..., True, True, True],[True, True, True, ..., True, True, True],...,[True, True, True, ..., True, True, True],[True, True, True, ..., True, True, True],[True, True, True, ..., True, True, True]],[[True, True, True, ..., True, True, True],[True, True, True, ..., True, True, True],[True, True, True, ..., True, True, True],...,[True, True, True, ..., True, True, True],[True, True, True, ..., True, True, True],[True, True, True, ..., True, True, True]],[[True, True, True, ..., True, True, True],[True, True, True, ..., True, True, True],[True, True, True, ..., True, True, True],...,[True, True, True, ..., True, True, True],[True, True, True, ..., True, True, True],[True, True, True, ..., True, True, True]]])

多种变化的组合transforms.Compose()

data_transforms &＃61; transforms.Compose([transforms.RandomResizedCrop(224),transforms.RandomHorizontalFlip(), # 以0.5的概率随机水平翻转transforms.ToTensor(),transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])


Dataset类的重写

Dataset类源码

class Dataset(object):"""An abstract class representing a Dataset.All other datasets should subclass it. All subclasses should override&＃96;&＃96;__len__&＃96;&＃96;, that provides the size of the dataset, and &＃96;&＃96;__getitem__&＃96;&＃96;,supporting integer indexing in range from 0 to len(self) exclusive."""def __getitem__(self, index):raise NotImplementedErrordef __len__(self):raise NotImplementedErrordef __add__(self, other):return ConcatDataset([self, other])


由上可以看出__getitem__和__len__方法必须重写
__len__方法用于使用 len(Dataset) 函数
__getitem__方法用于 Dataset[n] 操作
模板可以参考

from torch.utils.data import Dataset
import pandas as pd # pandas库提供了读取csv文件的函数read_csv()class myDataset(Dataset): # 定义自己的数据类myDataset&＃xff0c;继承的抽象类Datasetdef __init__(self, csv_file, txt_file,root_dir,other_file): # csv_file:抽象的表示.csv文件&＃xff1b;txt_file&＃xff1a;抽象的表示txt文件&＃xff1b;# root_dir&＃xff1a;地址&＃xff0c;这些参数放在初始化函数里self.csv_data&＃61; pd.read_csv(csv_file) # 读取csv文件&＃xff0c;并且赋给他本身with open(txt_file,&＃39;r&＃39;) as f: # 读取txt文件&＃xff0c;并且赋给他本身&＃xff0c;读取的方式为&＃xff1a;with open(...) as f:data_list &＃61; f.readlines() # 读取每一行数据&＃xff0c;并且放到data_list里self.txt_data &＃61; data_listself.root_dir &＃61; root_dir# 实现下面这个方法&＃xff1a; def __len__(self): # 定义自己的数据类&＃xff0c;必须重写这个方法&＃xff08;函数&＃xff09;return len(self.csv_data) # 返回的数据的长度def __getitem__(self, idx): # 定义自己的数据类&＃xff0c;必须重写这个方法&＃xff08;函数&＃xff09;data &＃61; (self.csv_data[idx],self.txt_data[idx]) # 获取数据的方式&＃xff0c;按照索引进行的 return data


数据的批量读取

torch.utils.data已经提供的类&＃xff1a;Dataset&＃xff0c;但是通过这种方式只能一个个的数据的把数据全部读出来&＃xff0c;定义了数据读取的方式&＃xff0c;不能实现** 批量**的把数据读取出来&＃xff0c;为此pytorch有提供了一个方法&＃xff1a;DataLoader&＃xff08;&＃xff09;&＃xff0c;它的参数如下&＃xff1a;

from torch.utils.data import DataLoaderdataiter &＃61; DataLoader(myDataset,batch_size&＃61;32,shuffle&＃61;True,collate_fn&＃61;default_collate)


myDatase&＃xff1a;上面自己定义的数据类
batch_size&＃61;32&＃xff1a;实现批量读取数据&＃xff0c;比如一次取32个数据
shuffle&＃61;True&＃xff1a;将顺序打乱
collate_fn&＃xff1a;表示的是如何读取样本&＃xff0c;可以自己定义函数来准确的说明想要实现的功能。
该段来源于该博客

以下是kaggle平台上猫狗识别数据集的制作

import numpy as np
from torchvision import transforms
from PIL import Image
from torch.utils.data import DataLoader, Dataset, ConcatDataset
import osclass catdogDataset(Dataset):def __init__(self, file_list, dir, mode&＃61;&＃39;train&＃39;, transform&＃61;None):self.file_list &＃61; file_listself.dir &＃61; dirself.mode &＃61; modeself.transform &＃61; transformif &＃39;dog&＃39; in self.file_list[0]:self.label &＃61; 1 # 标签为1时为狗&＃xff0c;标签为0是为猫else:self.label &＃61; 0def __len__(self):return len(self.file_list)def __getitem__(self, idx):img &＃61; Image.open(os.path.join(self.dir, self.file_list[idx]))if self.transform:img &＃61; self.transform(img)if self.mode &＃61;&＃61; &＃39;train&＃39;:img &＃61; np.array(img)return img.astype(&＃39;float32&＃39;), self.label # 此处当模式为训练时&＃xff0c;返回图片的32位浮点array和它的标签else:img &＃61; np.array(img)return img.astype(&＃39;float32&＃39;), self.file_list[idx]datapathtra &＃61; &＃39;./data/training_set/training_set&＃39;
cat_file_list &＃61; os.listdir(os.path.join(datapathtra, &＃39;cats&＃39;))
dog_file_list &＃61; os.listdir(os.path.join(datapathtra, &＃39;dogs&＃39;))data_transform &＃61; transforms.Compose([transforms.Resize([256, 256]),transforms.RandomHorizontalFlip(),transforms.ToTensor()
])catDataset &＃61; catdogDataset(cat_file_list, os.path.join(datapathtra, &＃39;cats&＃39;), transform&＃61;data_transform)
dogDataset &＃61; catdogDataset(dog_file_list, os.path.join(datapathtra, &＃39;dogs&＃39;), transform&＃61;data_transform)totalDataset &＃61; ConcatDataset([catDataset, dogDataset]) # 聚合两个数据集
print(totalDataset[5]) # 此处为__getitem__的用法
print(len(totalDataset)) # 此处为__len__的用法