Issues with running example "Training model for cars segmentation on CamVid dataset" using Pytorch 1.8.1

[zhigangmei]

Hi,

I have some issues while running example "Training model for cars segmentation on CamVid dataset" using Pytorch 1.8.1 with cuda11.1.

The error shows up when running "train_logs = train_epoch.run(train_loader)" inside the training loop.

Please let me know what I should do to solve the issue.

BTW, I have no issue to run the example on Colab in which Pytorch verson 1.8.0 with cuda10.1.

Thanks
Zhi-Gang

Error message looks like:

Empty Traceback (most recent call last)
~\anaconda3\envs\pytorch_1.8\lib\site-packages\torch\utils\data\dataloader.py in _try_get_data(self, timeout)
985 try:
--> 986 data = self._data_queue.get(timeout=timeout)
987 return (True, data)

~\anaconda3\envs\pytorch_1.8\lib\multiprocessing\queues.py in get(self, block, timeout)
107 if not self._poll(timeout):
--> 108 raise Empty
109 elif not self._poll():

Empty:

The above exception was the direct cause of the following exception:

RuntimeError Traceback (most recent call last)
in
6
7 print('\nEpoch: {}'.format(i))
----> 8 train_logs = train_epoch.run(train_loader)
9 valid_logs = valid_epoch.run(valid_loader)
10

~\anaconda3\envs\pytorch_1.8\lib\site-packages\segmentation_models_pytorch\utils\train.py in run(self, dataloader)
43
44 with tqdm(dataloader, desc=self.stage_name, file=sys.stdout, disable=not (self.verbose)) as iterator:
---> 45 for x, y in iterator:
46 x, y = x.to(self.device), y.to(self.device)
47 loss, y_pred = self.batch_update(x, y)

~\anaconda3\envs\pytorch_1.8\lib\site-packages\tqdm\std.py in iter(self)
1176
1177 try:
-> 1178 for obj in iterable:
1179 yield obj
1180 # Update and possibly print the progressbar.

~\anaconda3\envs\pytorch_1.8\lib\site-packages\torch\utils\data\dataloader.py in next(self)
515 if self._sampler_iter is None:
516 self._reset()
--> 517 data = self._next_data()
518 self._num_yielded += 1
519 if self._dataset_kind == _DatasetKind.Iterable and \

~\anaconda3\envs\pytorch_1.8\lib\site-packages\torch\utils\data\dataloader.py in _next_data(self)
1180
1181 assert not self._shutdown and self._tasks_outstanding > 0
-> 1182 idx, data = self._get_data()
1183 self._tasks_outstanding -= 1
1184 if self._dataset_kind == _DatasetKind.Iterable:

~\anaconda3\envs\pytorch_1.8\lib\site-packages\torch\utils\data\dataloader.py in _get_data(self)
1146 else:
1147 while True:
-> 1148 success, data = self._try_get_data()
1149 if success:
1150 return data

~\anaconda3\envs\pytorch_1.8\lib\site-packages\torch\utils\data\dataloader.py in _try_get_data(self, timeout)
997 if len(failed_workers) > 0:
998 pids_str = ', '.join(str(w.pid) for w in failed_workers)
--> 999 raise RuntimeError('DataLoader worker (pid(s) {}) exited unexpectedly'.format(pids_str)) from e
1000 if isinstance(e, queue.Empty):
1001 return (False, None)

RuntimeError: DataLoader worker (pid(s) 23504, 21820, 22636, 16552) exited unexpectedly