Terminate called after throwing an instance of ‘c10::CUDAError’ what(): CUDA error: initialization error

[thepurpleowl]

I am trying to run my pytorch-lghtning code on TPU in GCP.

import numpy as np # linear algebra
import pandas as pd 
import os
import string
from typing import Optional

# for checkpoint
# https://pytorch-lightning.readthedocs.io/en/latest/common/weights_loading.html
from torch.utils.data import DataLoader, Dataset, random_split
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim

import pytorch_lightning as pl
from pytorch_lightning import LightningDataModule
from pytorch_lightning.callbacks.early_stopping import EarlyStopping
from pytorch_lightning.callbacks import ModelCheckpoint

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import PCA
from sklearn.preprocessing import LabelEncoder, StandardScaler

import logging
logging.getLogger("lightning").setLevel(logging.ERROR)



class ModelDataset(Dataset):
    def __init__(self, X, y):
        self.X = X
        self.y = y
        self.len = len(self.X)
        
    def __getitem__(self, index):
        return self.X[index], self.y[index]
    
    def __len__(self):
        return self.len

    
class ModelDataModule(LightningDataModule):
    def __init__(self):
        super().__init__()
        self.train_root_path = './20news-bydate-train'
        self.test_root_path = './20news-bydate-test'
        self.batch_size = 64

    def get_data(self, root_path):
        data = []
        label = []

        folders = ['rec.sport.hockey', 'sci.electronics' , 'rec.autos']
        for i, folder_name in enumerate(folders):
            files = []
            folder_path = os.path.join(root_path, folder_name)
            files.extend([f for f in os.listdir(folder_path)])
            for file in files:
                file_path = os.path.join(folder_path, file)
                with open(file_path, 'r', encoding='latin1') as f:
                    content = f.read()
                    data.append(content)
                    label.append(i)
        return data, label

    def preprocess(self, words):
        table = str.maketrans('', '', '\t')
        words = [word.translate(table) for word in words]
        n_table = str.maketrans('', '', '\n')
        words = [word.translate(n_table) for word in words]

        punctuations = (string.punctuation).replace("'", "")
        trans_table = str.maketrans('', '', punctuations)
        stripped_words = [word.translate(trans_table) for word in words]

        words = [str for str in stripped_words if str]

        p_words = []
        for word in words:
            if (word[0] and word[len(word)-1] == "'"):
                word = word[1:len(word)-1]
            elif(word[0] == "'"):
                word = word[1:len(word)]
            else:
                word = word
            p_words.append(word)

        words = p_words.copy()
        words = [word for word in words if not word.isdigit()]
        words = [word for word in words if not len(word) == 1]
        words = [str for str in words if str]
        words = [word.lower() for word in words]
        words = [word for word in words if len(word) > 2]

        return " ".join(words)
    
    def prepare_data(self):
        ### called only on 1 GPU
        # get data
        train_data, train_label = self.get_data(self.train_root_path)
        test_data, test_label = self.get_data(self.test_root_path)

        # preprocess
        for i, text in enumerate(train_data):
            train_data[i] = self.preprocess(text.strip().split())

        for i, text in enumerate(test_data):
            test_data[i] = self.preprocess(text.strip().split())

        # feature engineering
        ### data
        vectorizer = TfidfVectorizer(max_df=0.75,stop_words='english')
        X_train = vectorizer.fit_transform(train_data)
        X_test = vectorizer.transform(test_data)

        X_train = X_train.toarray()
        X_test = X_test.toarray()

        pca_1k = PCA(n_components=1024)
        X_train1k = pca_1k.fit_transform(X_train)
        X_test1k = pca_1k.transform(X_test)
        print (pca_1k.explained_variance_ratio_.cumsum()[-1])
        
        ### label
        le = LabelEncoder()
        y_train = le.fit_transform(train_label)
        y_test = le.transform(test_label)
        
        # scaling
        scaler = StandardScaler()
        scaler.fit(X_train1k)
        
        self.train_data = torch.tensor(scaler.transform(X_train1k))
        self.test_data = torch.tensor(scaler.transform(X_test1k))
        self.train_label = torch.tensor(y_train)
        self.test_label = torch.tensor(y_test)
        

    def setup(self, stage: Optional[str] = None):
        # called on every GPU  
        train_data = ModelDataset(self.train_data, self.train_label)
        test_dataset = ModelDataset(self.test_data, self.test_label)
        train_len = (len(train_data)//10)*8
        val_len = len(train_data) - train_len
        print(train_len, val_len)
        train_dataset, val_dataset = random_split(train_data, [train_len, val_len],
                                                  generator=torch.Generator().manual_seed(42))

        self.train = train_dataset
        self.val = val_dataset
        self.test = test_dataset

    def train_dataloader(self):
        train_loader = DataLoader(dataset=self.train, 
                          batch_size=self.batch_size, shuffle=True, drop_last=True)
        return train_loader

    def val_dataloader(self):
        val_loader = DataLoader(dataset=self.val, 
                         batch_size=self.batch_size)
        return val_loader

    def test_dataloader(self):
        test_loader = DataLoader(dataset=self.test, 
                         batch_size=self.batch_size)
        return test_loader

class LightningFFModel(pl.LightningModule):
    def __init__(self, input_size, hidden_size, learning_rate):
        super().__init__()
        self.input_size = input_size
        self.hidden_size  = hidden_size
        self.fc1 = torch.nn.Linear(self.input_size, self.hidden_size)
        self.relu = torch.nn.ReLU()
        self.fc2 = torch.nn.Linear(self.hidden_size, 3)
        
        self.criterion = nn.CrossEntropyLoss()
        self.lr = learning_rate

    def forward(self, x):
        hidden = self.fc1(x)
        relu = self.relu(hidden)
        output = self.fc2(relu)
        return output

    def cross_entropy_loss(self, outputs, labels):
        return self.criterion(outputs, labels)

    def training_step(self, train_batch, batch_idx):
        x, y = train_batch
        outputs = self.forward(x.float())
        loss = self.cross_entropy_loss(outputs, y)
        self.log('train_loss', loss, rank_zero_only=True)
        return loss

    def validation_step(self, val_batch, batch_idx):
        x, y = val_batch
        outputs = self.forward(x.float())
        loss = self.cross_entropy_loss(outputs, y)
        self.log('val_loss', loss, rank_zero_only=True)

    def test_step(self, batch, batch_idx):
        x, y = batch
        outputs = self.forward(x.float())
        loss = self.cross_entropy_loss(outputs, y)
        self.log("test_loss", loss, rank_zero_only=True)

    def configure_optimizers(self):
        optimizer = optim.SGD(self.parameters(),
                              lr=(self.lr or self.learning_rate), momentum=0.9)
        return optimizer    
    

if __name__=='__main__':
	input_size = 1024
	hidden_size = 128
	model_dm = ModelDataModule()

	# train
	model = LightningFFModel(input_size, hidden_size, 0.005)

	early_stop_callback = EarlyStopping(monitor="val_loss", min_delta=0.00, patience=3, verbose=False, mode="min")
	checkpoint_callback = ModelCheckpoint(
		        monitor='val_loss',
		        dirpath='pt_checkpoints',
		        filename='epoch{epoch:02d}',
		        auto_insert_metric_name=False)

	trainer = pl.Trainer(tpu_cores=8, max_epochs=5,
		                 callbacks=[early_stop_callback, checkpoint_callback],
		                 auto_lr_find=True,
						 log_every_n_steps=1)


	trainer.fit(model, datamodule=model_dm)

I am getting the error

terminate called after throwing an instance of 'c10::CUDAError' what(): CUDA error: initialization error CUDA kernel errors might be asynchronously reported at some other API call,so the stacktrace below might be incorrect. For debugging consider passing CUDA_LAUNCH_BLOCKING=1. ... torch.multiprocessing.spawn.ProcessExitedException: process 7 terminated with signal SIGABRT

The full stack trace is as follows,

GPU available: True, used: False
TPU available: True, using: 8 TPU cores
IPU available: False, using: 0 IPUs
/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py:1296: UserWarning: GPU available but not used. Set the gpus flag in your trainer `Trainer(gpus=1)` or script `--gpus=1`.
  "GPU available but not used. Set the gpus flag in your trainer"
Validation sanity check: 0it [00:00, ?it/s]/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/data_loading.py:106: UserWarning: The dataloader, val dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 8 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.
  f"The dataloader, {name}, does not have many workers which may be a bottleneck."
/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/data_loading.py:106: UserWarning: The dataloader, train dataloader, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 8 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.
  f"The dataloader, {name}, does not have many workers which may be a bottleneck."
Epoch 0:   0%|                                                                     | 0/3 [00:00<00:00, 2368.33it/s]terminate called after throwing an instance of 'c10::CUDAError'
  what():  CUDA error: initialization error
CUDA kernel errors might be asynchronously reported at some other API call,so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1.
Exception raised from getDevice at /opt/conda/conda-bld/pytorch_1623448265233/work/c10/cuda/impl/CUDAGuardImpl.h:38 (most recent call first):
frame #0: c10::Error::Error(c10::SourceLocation, std::string) + 0x42 (0x7fbac23cfa22 in /opt/conda/lib/python3.7/site-packages/torch/lib/libc10.so)
frame #1: <unknown function> + 0x9b5b (0x7fbac262ab5b in /opt/conda/lib/python3.7/site-packages/torch/lib/libc10_cuda.so)
frame #2: <unknown function> + 0x3280aaa (0x7fbac5ac9aaa in /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #3: torch::autograd::Engine::thread_init(int, std::shared_ptr<torch::autograd::ReadyQueue> const&, bool) + 0x2a (0x7fbac5acabfa in /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #4: torch::autograd::python::PythonEngine::thread_init(int, std::shared_ptr<torch::autograd::ReadyQueue> const&, bool) + 0x53 (0x7fbb3bc9dc43 in /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch_python.so)
frame #5: <unknown function> + 0xc9067 (0x7fbb70d75067 in /opt/conda/lib/python3.7/site-packages/pyarrow/../../../libstdc++.so.6)
frame #6: <unknown function> + 0x76db (0x7fbb8340f6db in /lib/x86_64-linux-gnu/libpthread.so.0)
frame #7: clone + 0x3f (0x7fbb8278b71f in /lib/x86_64-linux-gnu/libc.so.6)

terminate called after throwing an instance of 'terminate called after throwing an instance of 'c10::CUDAErrorc10::CUDAError'
'
  what():    what():  CUDA error: initialization error
CUDA kernel errors might be asynchronously reported at some other API call,so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1.
Exception raised from getDevice at /opt/conda/conda-bld/pytorch_1623448265233/work/c10/cuda/impl/CUDAGuardImpl.h:38 (most recent call first):
frame #0: c10::Error::Error(c10::SourceLocation, std::string) + 0x42 (0x7fbac23cfa22 in /opt/conda/lib/python3.7/site-packages/torch/lib/libc10.so)
frame #1: <unknown function> + 0x9b5b (0x7fbac262ab5b in /opt/conda/lib/python3.7/site-packages/torch/lib/libc10_cuda.so)
frame #2: <unknown function> + 0x3280aaa (0x7fbac5ac9aaa in /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #3: torch::autograd::Engine::thread_init(int, std::shared_ptr<torch::autograd::ReadyQueue> const&, bool) + 0x2a (0x7fbac5acabfa in /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #4: torch::autograd::python::PythonEngine::thread_init(int, std::shared_ptr<torch::autograd::ReadyQueue> const&, bool) + 0x53 (0x7fbb3bc9dc43 in /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch_python.so)
frame #5: <unknown function> + 0xc9067 (0x7fbb70d75067 in /opt/conda/lib/python3.7/site-packages/pyarrow/../../../libstdc++.so.6)
frame #6: <unknown function> + 0x76db (0x7fbb8340f6db in /lib/x86_64-linux-gnu/libpthread.so.0)
frame #7: clone + 0x3f (0x7fbb8278b71f in /lib/x86_64-linux-gnu/libc.so.6)

CUDA error: initialization error
CUDA kernel errors might be asynchronously reported at some other API call,so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1.
Exception raised from getDevice at /opt/conda/conda-bld/pytorch_1623448265233/work/c10/cuda/impl/CUDAGuardImpl.h:38 (most recent call first):
frame #0: c10::Error::Error(c10::SourceLocation, std::string) + 0x42 (0x7fbac23cfa22 in /opt/conda/lib/python3.7/site-packages/torch/lib/libc10.so)
frame #1: <unknown function> + 0x9b5b (0x7fbac262ab5b in /opt/conda/lib/python3.7/site-packages/torch/lib/libc10_cuda.so)
frame #2: <unknown function> + 0x3280aaa (0x7fbac5ac9aaa in /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #3: torch::autograd::Engine::thread_init(int, std::shared_ptr<torch::autograd::ReadyQueue> const&, bool) + 0x2a (0x7fbac5acabfa in /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #4: torch::autograd::python::PythonEngine::thread_init(int, std::shared_ptr<torch::autograd::ReadyQueue> const&, bool) + 0x53 (0x7fbb3bc9dc43 in /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch_python.so)
frame #5: <unknown function> + 0xc9067 (0x7fbb70d75067 in /opt/conda/lib/python3.7/site-packages/pyarrow/../../../libstdc++.so.6)
frame #6: <unknown function> + 0x76db (0x7fbb8340f6db in /lib/x86_64-linux-gnu/libpthread.so.0)
frame #7: clone + 0x3f (0x7fbb8278b71f in /lib/x86_64-linux-gnu/libc.so.6)

terminate called after throwing an instance of 'c10::CUDAError'
  what():  CUDA error: initialization error
CUDA kernel errors might be asynchronously reported at some other API call,so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1.
Exception raised from getDevice at /opt/conda/conda-bld/pytorch_1623448265233/work/c10/cuda/impl/CUDAGuardImpl.h:38 (most recent call first):
frame #0: c10::Error::Error(c10::SourceLocation, std::string) + 0x42 (0x7fbac23cfa22 in /opt/conda/lib/python3.7/site-packages/torch/lib/libc10.so)
frame #1: <unknown function> + 0x9b5b (0x7fbac262ab5b in /opt/conda/lib/python3.7/site-packages/torch/lib/libc10_cuda.so)
frame #2: <unknown function> + 0x3280aaa (0x7fbac5ac9aaa in /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #3: torch::autograd::Engine::thread_init(int, std::shared_ptr<torch::autograd::ReadyQueue> const&, bool) + 0x2a (0x7fbac5acabfa in /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #4: torch::autograd::python::PythonEngine::thread_init(int, std::shared_ptr<torch::autograd::ReadyQueue> const&, bool) + 0x53 (0x7fbb3bc9dc43 in /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch_python.so)
frame #5: <unknown function> + 0xc9067 (0x7fbb70d75067 in /opt/conda/lib/python3.7/site-packages/pyarrow/../../../libstdc++.so.6)
frame #6: <unknown function> + 0x76db (0x7fbb8340f6db in /lib/x86_64-linux-gnu/libpthread.so.0)
frame #7: clone + 0x3f (0x7fbb8278b71f in /lib/x86_64-linux-gnu/libc.so.6)

terminate called after throwing an instance of 'c10::CUDAError'
  what():  CUDA error: initialization error
CUDA kernel errors might be asynchronously reported at some other API call,so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1.
Exception raised from getDevice at /opt/conda/conda-bld/pytorch_1623448265233/work/c10/cuda/impl/CUDAGuardImpl.h:38 (most recent call first):
frame #0: c10::Error::Error(c10::SourceLocation, std::string) + 0x42 (0x7fbac23cfa22 in /opt/conda/lib/python3.7/site-packages/torch/lib/libc10.so)
frame #1: <unknown function> + 0x9b5b (0x7fbac262ab5b in /opt/conda/lib/python3.7/site-packages/torch/lib/libc10_cuda.so)
frame #2: <unknown function> + 0x3280aaa (0x7fbac5ac9aaa in /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #3: torch::autograd::Engine::thread_init(int, std::shared_ptr<torch::autograd::ReadyQueue> const&, bool) + 0x2a (0x7fbac5acabfa in /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #4: torch::autograd::python::PythonEngine::thread_init(int, std::shared_ptr<torch::autograd::ReadyQueue> const&, bool) + 0x53 (0x7fbb3bc9dc43 in /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch_python.so)
frame #5: <unknown function> + 0xc9067 (0x7fbb70d75067 in /opt/conda/lib/python3.7/site-packages/pyarrow/../../../libstdc++.so.6)
frame #6: <unknown function> + 0x76db (0x7fbb8340f6db in /lib/x86_64-linux-gnu/libpthread.so.0)
frame #7: clone + 0x3f (0x7fbb8278b71f in /lib/x86_64-linux-gnu/libc.so.6)

terminate called after throwing an instance of 'c10::CUDAError'
  what():  CUDA error: initialization error
CUDA kernel errors might be asynchronously reported at some other API call,so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1.
Exception raised from getDevice at /opt/conda/conda-bld/pytorch_1623448265233/work/c10/cuda/impl/CUDAGuardImpl.h:38 (most recent call first):
frame #0: c10::Error::Error(c10::SourceLocation, std::string) + 0x42 (0x7fbac23cfa22 in /opt/conda/lib/python3.7/site-packages/torch/lib/libc10.so)
frame #1: <unknown function> + 0x9b5b (0x7fbac262ab5b in /opt/conda/lib/python3.7/site-packages/torch/lib/libc10_cuda.so)
frame #2: <unknown function> + 0x3280aaa (0x7fbac5ac9aaa in /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #3: torch::autograd::Engine::thread_init(int, std::shared_ptr<torch::autograd::ReadyQueue> const&, bool) + 0x2a (0x7fbac5acabfa in /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #4: torch::autograd::python::PythonEngine::thread_init(int, std::shared_ptr<torch::autograd::ReadyQueue> const&, bool) + 0x53 (0x7fbb3bc9dc43 in /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch_python.so)
frame #5: <unknown function> + 0xc9067 (0x7fbb70d75067 in /opt/conda/lib/python3.7/site-packages/pyarrow/../../../libstdc++.so.6)
frame #6: <unknown function> + 0x76db (0x7fbb8340f6db in /lib/x86_64-linux-gnu/libpthread.so.0)
frame #7: clone + 0x3f (0x7fbb8278b71f in /lib/x86_64-linux-gnu/libc.so.6)

terminate called after throwing an instance of 'c10::CUDAError'
  what():  CUDA error: initialization error
CUDA kernel errors might be asynchronously reported at some other API call,so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1.
Exception raised from getDevice at /opt/conda/conda-bld/pytorch_1623448265233/work/c10/cuda/impl/CUDAGuardImpl.h:38 (most recent call first):
frame #0: c10::Error::Error(c10::SourceLocation, std::string) + 0x42 (0x7fbac23cfa22 in /opt/conda/lib/python3.7/site-packages/torch/lib/libc10.so)
frame #1: <unknown function> + 0x9b5b (0x7fbac262ab5b in /opt/conda/lib/python3.7/site-packages/torch/lib/libc10_cuda.so)
frame #2: <unknown function> + 0x3280aaa (0x7fbac5ac9aaa in /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #3: torch::autograd::Engine::thread_init(int, std::shared_ptr<torch::autograd::ReadyQueue> const&, bool) + 0x2a (0x7fbac5acabfa in /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #4: torch::autograd::python::PythonEngine::thread_init(int, std::shared_ptr<torch::autograd::ReadyQueue> const&, bool) + 0x53 (0x7fbb3bc9dc43 in /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch_python.so)
frame #5: <unknown function> + 0xc9067 (0x7fbb70d75067 in /opt/conda/lib/python3.7/site-packages/pyarrow/../../../libstdc++.so.6)
frame #6: <unknown function> + 0x76db (0x7fbb8340f6db in /lib/x86_64-linux-gnu/libpthread.so.0)
frame #7: clone + 0x3f (0x7fbb8278b71f in /lib/x86_64-linux-gnu/libc.so.6)

terminate called after throwing an instance of 'c10::CUDAError'
  what():  CUDA error: initialization error
CUDA kernel errors might be asynchronously reported at some other API call,so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1.
Exception raised from getDevice at /opt/conda/conda-bld/pytorch_1623448265233/work/c10/cuda/impl/CUDAGuardImpl.h:38 (most recent call first):
frame #0: c10::Error::Error(c10::SourceLocation, std::string) + 0x42 (0x7fbac23cfa22 in /opt/conda/lib/python3.7/site-packages/torch/lib/libc10.so)
frame #1: <unknown function> + 0x9b5b (0x7fbac262ab5b in /opt/conda/lib/python3.7/site-packages/torch/lib/libc10_cuda.so)
frame #2: <unknown function> + 0x3280aaa (0x7fbac5ac9aaa in /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #3: torch::autograd::Engine::thread_init(int, std::shared_ptr<torch::autograd::ReadyQueue> const&, bool) + 0x2a (0x7fbac5acabfa in /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so)
frame #4: torch::autograd::python::PythonEngine::thread_init(int, std::shared_ptr<torch::autograd::ReadyQueue> const&, bool) + 0x53 (0x7fbb3bc9dc43 in /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch_python.so)
frame #5: <unknown function> + 0xc9067 (0x7fbb70d75067 in /opt/conda/lib/python3.7/site-packages/pyarrow/../../../libstdc++.so.6)
frame #6: <unknown function> + 0x76db (0x7fbb8340f6db in /lib/x86_64-linux-gnu/libpthread.so.0)
frame #7: clone + 0x3f (0x7fbb8278b71f in /lib/x86_64-linux-gnu/libc.so.6)

Traceback (most recent call last):
  File "pt_test.py", line 229, in <module>
    trainer.fit(model, datamodule=model_dm)
  File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 553, in fit
    self._run(model)
  File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 918, in _run
    self._dispatch()
  File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 986, in _dispatch
    self.accelerator.start_training(self)
  File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/accelerators/accelerator.py", line 92, in start_training
    self.training_type_plugin.start_training(trainer)
  File "/opt/conda/lib/python3.7/site-packages/pytorch_lightning/plugins/training_type/tpu_spawn.py", line 267, in start_training
    xmp.spawn(self.new_process, **self.xmp_spawn_kwargs)
  File "/opt/conda/lib/python3.7/site-packages/torch_xla/distributed/xla_multiprocessing.py", line 394, in spawn
    start_method=start_method)
  File "/opt/conda/lib/python3.7/site-packages/torch/multiprocessing/spawn.py", line 188, in start_processes
    while not context.join():
  File "/opt/conda/lib/python3.7/site-packages/torch/multiprocessing/spawn.py", line 136, in join
    signal_name=name
torch.multiprocessing.spawn.ProcessExitedException: process 7 terminated with signal SIGABRT

python: 3.7
pytorch: 1.9
pytorch-lightning: 1.4.4
cuda: 11.1
tpu: v2-8

This works on single GPU and Error occurs at trainer.fit
What does c10-cudaerror mean? Is it something related to cuda version 10?
Any ideas what's going wrong?

[callmeBalloch]

Ok, replying here (the forum seems not the way to go anymore)!
Look at these:
#8821
#4471

In my case is related to ddp: ##8821 (comment)

[carmocca]

Thanks for the links! Let's continue this on the tagged issues. Marking your comment as the answer

[thepurpleowl]

I am unmarking this as answer. As the issues are related to GPU, and I'm facing this with TPU pod training.
Suggested work arounds such as the #8821 comment doesn't solve this.
I'll mark it answered when this issue gets resolved.

@carmocca Should I create a new issue?

[tchaton]

Dear @thepurpleowl,

It seems your machine has both cuda and TPU available. Since there's the issue linked above with cuda, this might be why you are experiencing it for TPUs

GPU available: True, used: False
TPU available: True, using: 8 TPU cores
IPU available: False, using: 0 IPUs

Best,
T.C

[thepurpleowl]

So if I understand correctly, running the code on a VM without GPU will confirm this. I'll check.