{"id":186,"date":"2025-06-30T11:02:08","date_gmt":"2025-06-30T03:02:08","guid":{"rendered":"https:\/\/mianka.xyz\/?p=186"},"modified":"2025-06-30T11:18:01","modified_gmt":"2025-06-30T03:18:01","slug":"pytorchdataset","status":"publish","type":"post","link":"https:\/\/www.mianka.xyz\/?p=186","title":{"rendered":"PyTorch\u81ea\u5b9a\u4e49Dataset\u5168\u89e3\u6790\uff1a\u4ece\u7406\u8bba\u5230\u5b9e\u6218\u7684\u5b8c\u6574\u6307\u5357"},"content":{"rendered":"

\u4e00\u3001Dataset\u7684\u6838\u5fc3\u673a\u5236<\/h2>\n

PyTorch\u7684\u6570\u636e\u52a0\u8f7d\u4f53\u7cfb\u57fa\u4e8e\u4e24\u5927\u6838\u5fc3\u7ec4\u4ef6\uff1a<\/p>\n

    \n
  1. \n

    Dataset<\/span>\uff1a\u5b9a\u4e49\u6570\u636e\u96c6\u7684\u62bd\u8c61\u63a5\u53e3\uff0c\u8d1f\u8d23\u7d22\u5f15\u5230\u6837\u672c\u7684\u6620\u5c04\u3002<\/p>\n<\/li>\n

  2. \n

    DataLoader<\/span>\uff1a\u5c01\u88c5Dataset\uff0c\u63d0\u4f9b\u6279\u91cf\u52a0\u8f7d\u3001\u591a\u7ebf\u7a0b\u52a0\u901f\u7b49\u529f\u80fd\u3002<\/p>\n<\/li>\n<\/ol>\n

    1.1 \u6570\u636e\u4f20\u9012\u6d41\u7a0b<\/h3>\n
    # \u5178\u578b\u6d41\u7a0b\u793a\u4f8b\ndataset = CustomDataset(...)  # \u521b\u5efa\u81ea\u5b9a\u4e49\u6570\u636e\u96c6\ndataloader = DataLoader(dataset, batch_size=32, shuffle=True)  # \u5305\u88c5\u4e3aDataLoader\nfor batch in dataloader:  # \u8fed\u4ee3\u83b7\u53d6\u6279\u91cf\u6570\u636e\n    inputs, labels = batch<\/pre>\n
    1.2 Map\u5f0f vs Iterable\u5f0f\u6570\u636e\u96c6<\/h3>\n
      \n
    • \n
      Map\u5f0f\u6570\u636e\u96c6<\/span>\uff08\u5e38\u7528\uff09\uff1a\u901a\u8fc7__getitem__<\/code>\u5b9e\u73b0\u7d22\u5f15\u8bbf\u95ee\uff0c\u652f\u6301\u968f\u673a\u6253\u4e71\uff08shuffle\uff09\u3002<\/p>\n<\/li>\n
    • \n
      Iterable\u5f0f\u6570\u636e\u96c6<\/span>\uff1a\u9002\u7528\u4e8e\u6d41\u5f0f\u6570\u636e\uff08\u5982\u5b9e\u65f6\u4f20\u611f\u5668\u6570\u636e\uff09\uff0c\u6309\u987a\u5e8f\u8fed\u4ee3\u3002<\/p>\n<\/li>\n<\/ul>\n
      \u4e8c\u3001\u81ea\u5b9a\u4e49Dataset\u7684\u5b9e\u73b0\u8303\u5f0f<\/h2>\n
      \u7ee7\u627ftorch.utils.data.Dataset<\/code>\u9700\u5b9e\u73b0\u4e09\u4e2a\u6838\u5fc3\u65b9\u6cd5\uff1a<\/p>\n
      2.1 \u57fa\u7840\u6a21\u677f<\/h3>\n
      from torch.utils.data import Dataset\n\nclass CustomDataset(Dataset):\n    def __init__(self, data_path, transform=None):\n        self.data_path = data_path\n        self.transform = transform\n        self.samples = self._load_data()  # \u52a0\u8f7d\u6570\u636e\u5217\u8868\n\n    def __len__(self):\n        return len(self.samples)\n\n    def __getitem__(self, index):\n        # 1. \u8bfb\u53d6\u539f\u59cb\u6570\u636e\n        sample = self._read_sample(index)\n        \n        # 2. \u5e94\u7528\u9884\u5904\u7406\n        if self.transform:\n            sample = self.transform(sample)\n            \n        # 3. \u8fd4\u56de\u6837\u672c\uff08\u5982\u56fe\u50cf+\u6807\u7b7e\uff09\n        return sample<\/pre>\n
      2.2 \u5173\u952e\u65b9\u6cd5\u8be6\u89e3<\/h3>\n
        \n
      • \n
        __init__<\/code><\/span>\uff1a\u521d\u59cb\u5316\u6570\u636e\u8def\u5f84\u3001\u9884\u5904\u7406\u53d8\u6362\uff0c\u5e76\u52a0\u8f7d\u6570\u636e\u5143\u4fe1\u606f\u3002<\/p>\n<\/li>\n
      • \n
        __len__<\/code><\/span>\uff1a\u8fd4\u56de\u6570\u636e\u96c6\u5927\u5c0f\uff0c\u7528\u4e8eDataLoader\u7684\u8fdb\u5ea6\u63a7\u5236\u3002<\/p>\n<\/li>\n
      • \n
        __getitem__<\/code><\/span>\uff1a\u6838\u5fc3\u65b9\u6cd5\uff0c\u9700\u5b8c\u6210\u6570\u636e\u8bfb\u53d6\u3001\u9884\u5904\u7406\u548c\u8fd4\u56de\u3002<\/p>\n<\/li>\n<\/ul>\n
        \u4e09\u3001\u5b9e\u6218\u6848\u4f8b\uff1a\u56fe\u50cf\u5206\u7c7b\u6570\u636e\u96c6<\/h2>\n
        \u4ee5Kaggle\u7684Dogs vs Cats\u6570\u636e\u96c6\u4e3a\u4f8b\uff0c\u5b9e\u73b0\u81ea\u5b9a\u4e49Dataset\uff1a<\/p>\n
        3.1 \u6570\u636e\u51c6\u5907<\/h3>\n
        dataset\/\n\u251c\u2500\u2500 train\/\n\u2502   \u251c\u2500\u2500 cat.0.jpg\n\u2502   \u251c\u2500\u2500 dog.0.jpg\n\u2502   \u2514\u2500\u2500 ...\n\u2514\u2500\u2500 annotations.txt  # \u683c\u5f0f\uff1afilename label<\/pre>\n
        3.2 \u5b8c\u6574\u5b9e\u73b0<\/h3>\n
        import os\nfrom PIL import Image\nfrom torch.utils.data import Dataset\nfrom torchvision import transforms\n\nclass DogCatDataset(Dataset):\n    def __init__(self, root_dir, ann_file, transform=None):\n        self.root_dir = root_dir\n        self.transform = transform\n        self.samples = self._load_annotations(ann_file)\n\n    def _load_annotations(self, ann_file):\n        samples = []\n        with open(ann_file) as f:\n            for line in f:\n                filename, label = line.strip().split()\n                samples.append((filename, int(label)))\n        return samples\n\n    def __len__(self):\n        return len(self.samples)\n\n    def __getitem__(self, index):\n        filename, label = self.samples[index]\n        img_path = os.path.join(self.root_dir, filename)\n        image = Image.open(img_path)\n        \n        if self.transform:\n            image = self.transform(image)\n            \n        return image, label\n\n# \u4f7f\u7528\u793a\u4f8b\ntransform = transforms.Compose([\n    transforms.Resize(256),\n    transforms.CenterCrop(224),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n])\n\ndataset = DogCatDataset(\n    root_dir='dataset\/train',\n    ann_file='dataset\/annotations.txt',\n    transform=transform\n)\n\ndataloader = DataLoader(dataset, batch_size=32, shuffle=True, num_workers=4)<\/pre>\n
        <\/span><\/p>\n
        \u56db\u3001\u9ad8\u7ea7\u6280\u5de7\u4e0e\u4f18\u5316<\/h2>\n
        4.1 \u591a\u7ebf\u7a0b\u52a0\u901f<\/h3>\n
        \u901a\u8fc7num_workers<\/code>\u53c2\u6570\u542f\u7528\u591a\u8fdb\u7a0b\u52a0\u8f7d\uff1a<\/p>\n
        pythondataloader = DataLoader(dataset, batch_size=64, num_workers=8)<\/pre>\n
        4.2 \u81ea\u5b9a\u4e49Collate\u51fd\u6570<\/h3>\n
        \u5904\u7406\u53d8\u957f\u6570\u636e\uff08\u5982NLP\u5e8f\u5217\uff09\uff1a<\/p>\n
        def collate_fn(batch):\n    # batch: List[Tuple(image, label)]\n    images = [item[0] for item in batch]\n    labels = [item[1] for item in batch]\n    # \u5806\u53e0\u56fe\u50cf\u5e76\u8f6c\u6362\u4e3aTensor\n    images = torch.stack(images, dim=0)\n    labels = torch.LongTensor(labels)\n    return images, labels\n\ndataloader = DataLoader(dataset, batch_size=32, collate_fn=collate_fn)<\/pre>\n
        <\/span><\/p>\n
        4.3 \u5185\u5b58\u6620\u5c04\u4f18\u5316<\/h3>\n
        \u5bf9\u4e8e\u5927\u578b\u6570\u636e\u96c6\uff0c\u53ef\u4f7f\u7528\u5185\u5b58\u6620\u5c04\u6587\u4ef6\uff08\u5982HDF5\uff09\u51cf\u5c11I\/O\u5f00\u9500\uff1a<\/p>\n
        import h5py\n\nclass HDF5Dataset(Dataset):\n    def __init__(self, h5_path):\n        self.h5_file = h5py.File(h5_path, 'r')\n        self.length = len(self.h5_file['images'])\n\n    def __getitem__(self, index):\n        image = self.h5_file['images'][index]\n        label = self.h5_file['labels'][index]\n        return image, label<\/pre>\n
        \u4e94\u3001\u5e38\u89c1\u95ee\u9898\u4e0e\u89e3\u51b3\u65b9\u6848<\/h2>\n
        5.1 \u6570\u636e\u8def\u5f84\u9519\u8bef<\/h3>\n
          \n
        • \n
          \u95ee\u9898<\/span>\uff1aFileNotFoundError<\/code><\/p>\n<\/li>\n
        • \n
          \u89e3\u51b3<\/span>\uff1a\u4f7f\u7528os.path.join<\/code>\u6784\u5efa\u8de8\u5e73\u53f0\u8def\u5f84\uff0c\u68c0\u67e5\u6587\u4ef6\u6743\u9650\u3002<\/p>\n<\/li>\n<\/ul>\n
          5.2 \u5185\u5b58\u4e0d\u8db3<\/h3>\n
            \n
          • \n
            \u95ee\u9898<\/span>\uff1a\u52a0\u8f7d\u5927\u578b\u6570\u636e\u96c6\u65f6OOM<\/p>\n<\/li>\n
          • \n
            \u89e3\u51b3<\/span>\uff1a<\/p>\n<\/li>\n
              \n
            • \n
              \u4f7f\u7528\u751f\u6210\u5668\uff08IterableDataset\uff09<\/p>\n<\/li>\n
            • \n
              \u5206\u6279\u52a0\u8f7d\u6570\u636e<\/p>\n<\/li>\n
            • \n
              \u964d\u4f4ebatch_size<\/code><\/p>\n<\/li>\n<\/ul>\n<\/ul>\n
              5.3 \u6570\u636e\u7c7b\u578b\u4e0d\u5339\u914d<\/h3>\n
                \n
              • \n
                \u95ee\u9898<\/span>\uff1aRuntimeError: Input type (torch.FloatTensor) and weight type (torch.cuda.FloatTensor)<\/code><\/p>\n<\/li>\n
              • \n
                \u89e3\u51b3<\/span>\uff1a\u663e\u5f0f\u8c03\u7528.to(device)<\/code>\u6216\u4f7f\u7528pin_memory=True<\/code>\u52a0\u901fGPU\u4f20\u8f93\u3002<\/p>\n<\/li>\n<\/ul>\n
                \u516d\u3001\u603b\u7ed3<\/h2>\n
                \u81ea\u5b9a\u4e49Dataset\u662fPyTorch\u6570\u636e\u6d41\u6c34\u7ebf\u7684\u6838\u5fc3\u6280\u80fd\uff0c\u901a\u8fc7\u7ee7\u627fDataset<\/code>\u7c7b\u5e76\u5b9e\u73b0__len__<\/code>\u548c__getitem__<\/code>\u65b9\u6cd5\uff0c\u5f00\u53d1\u8005\u53ef\u4ee5\u7075\u6d3b\u5904\u7406\u5404\u7c7b\u6570\u636e\u683c\u5f0f\u3002\u7ed3\u5408DataLoader<\/code>\u7684\u591a\u7ebf\u7a0b\u52a0\u901f\u548c\u81ea\u5b9a\u4e49collate_fn<\/code>\uff0c\u53ef\u6784\u5efa\u9ad8\u6548\u7684\u6570\u636e\u52a0\u8f7d\u7ba1\u9053\u3002\u5b9e\u9645\u5e94\u7528\u4e2d\u9700\u6ce8\u610f\u8def\u5f84\u5904\u7406\u3001\u5185\u5b58\u4f18\u5316\u548c\u8bbe\u5907\u517c\u5bb9\u6027\uff0c\u4ee5\u786e\u4fdd\u8bad\u7ec3\u8fc7\u7a0b\u7684\u7a33\u5b9a\u6027\u3002<\/p>\n
                <\/p>\n","protected":false},"excerpt":{"rendered":"
                <\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[3],"tags":[],"class_list":["post-186","post","type-post","status-publish","format-standard","hentry","category-pythonbiji"],"_links":{"self":[{"href":"https:\/\/www.mianka.xyz\/index.php?rest_route=\/wp\/v2\/posts\/186","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.mianka.xyz\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.mianka.xyz\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.mianka.xyz\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/www.mianka.xyz\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=186"}],"version-history":[{"count":0,"href":"https:\/\/www.mianka.xyz\/index.php?rest_route=\/wp\/v2\/posts\/186\/revisions"}],"wp:attachment":[{"href":"https:\/\/www.mianka.xyz\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=186"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.mianka.xyz\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=186"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.mianka.xyz\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=186"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}