Code help for Simple RNN text sequence

[sushmit86]

I am new to RNN and Pytorch lightning and was trying to convert a simple RNN to lightning. Here is the regular Pytorch code

### Regular Pytorch Implementation

import pandas as pd
import numpy as np
import torch
from torch import nn
text = ['hey how are you','good i am fine','have a nice day']
chars  = set(''.join(text))
print(chars)
int2char = dict(enumerate(chars))
print(int2char)
char2int = {char: ind for ind, char in int2char.items()}
for i in range(len(text)):
    while len(text[i])<maxlen:
        text[i] += ' '

# Creating lists that will hold our input and target sequences
input_seq = []
target_seq = []

for i in range(len(text)):
    # Remove last character for input sequence
    input_seq.append(text[i][:-1])
    
    # Remove firsts character for target sequence
    target_seq.append(text[i][1:])
    print("Input Sequence: {}\nTarget Sequence: {}".format(input_seq[i], target_seq[i]))
for i in range(len(text)):
    input_seq[i] = [char2int[character] for character in input_seq[i]]
    target_seq[i] = [char2int[character] for character in target_seq[i]]

dict_size = len(char2int)
seq_len = maxlen - 1
batch_size = len(text)

def one_hot_encode(sequence, dict_size, seq_len, batch_size):
    # Creating a multi-dimensional array of zeros with the desired output shape
    features = np.zeros((batch_size, seq_len, dict_size), dtype=np.float32)
    
    # Replacing the 0 at the relevant character index with a 1 to represent that character
    for i in range(batch_size):
        for u in range(seq_len):
            features[i, u, sequence[i][u]] = 1
    return features
input_seq = one_hot_encode(input_seq, dict_size, seq_len, batch_size)
print("Input shape: {} --> (Batch Size, Sequence Length, One-Hot Encoding Size)".format(input_seq.shape))
input_seq = torch.from_numpy(input_seq)
target_seq = torch.Tensor(target_seq)
# torch.cuda.is_available() checks and returns a Boolean True if a GPU is available, else it'll return False
is_cuda = torch.cuda.is_available()

# If we have a GPU available, we'll set our device to GPU. We'll use this device variable later in our code.
if is_cuda:
    device = torch.device("cuda")
    print("GPU is available")
else:
    device = torch.device("cpu")
    print("GPU not available, CPU used")
class Model(nn.Module):
    def __init__(self, input_size, output_size, hidden_dim, n_layers):
        super(Model, self).__init__()

        # Defining some parameters
        self.hidden_dim = hidden_dim
        self.n_layers = n_layers

        #Defining the layers
        # RNN Layer
        self.rnn = nn.RNN(input_size, hidden_dim, n_layers, batch_first=True)   
        # Fully connected layer
        self.fc = nn.Linear(hidden_dim, output_size)
    
    def forward(self, x):
        
        batch_size = x.size(0)

        #Initializing hidden state for first input using method defined below
        hidden = self.init_hidden(batch_size)

        # Passing in the input and hidden state into the model and obtaining outputs
        out, hidden = self.rnn(x, hidden)
        
        # Reshaping the outputs such that it can be fit into the fully connected layer
        out = out.contiguous().view(-1, self.hidden_dim)
        out = self.fc(out)
        
        return out, hidden
    
    def init_hidden(self, batch_size):
        # This method generates the first hidden state of zeros which we'll use in the forward pass
        hidden = torch.zeros(self.n_layers, batch_size, self.hidden_dim).to(device)
         # We'll send the tensor holding the hidden state to the device we specified earlier as well
        return hidden
# Instantiate the model with hyperparameters
model = Model(input_size=dict_size, output_size=dict_size, hidden_dim=12, n_layers=1)
# We'll also set the model to the device that we defined earlier (default is CPU)
model = model.to(device)

# Define hyperparameters
n_epochs = 100
lr=0.01

# Define Loss, Optimizer
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
# Training Run
input_seq = input_seq.to(device)
for epoch in range(1, n_epochs + 1):
    optimizer.zero_grad() # Clears existing gradients from previous epoch
    #input_seq = input_seq.to(device)
    output, hidden = model(input_seq)
    output = output.to(device)
    target_seq = target_seq.to(device)
    loss = criterion(output, target_seq.view(-1).long())
    loss.backward() # Does backpropagation and calculates gradients
    optimizer.step() # Updates the weights accordingly
    
    if epoch%10 == 0:
        print('Epoch: {}/{}.............'.format(epoch, n_epochs), end=' ')
        print("Loss: {:.4f}".format(loss.item()))

and this is what I was trying:

import pandas as pd
import torch
import numpy as np
import pytorch_lightning as pl
import torchmetrics
from torchvision.datasets import FashionMNIST
from torch.utils.data import DataLoader
from torchvision import transforms
from torch.nn import functional as F
import mlflow

class Lit_Datamodule(pl.LightningDataModule):
  def __init__(self,input_text = ['hey how are you','good i am fine','have a nice day']):
    super().__init__()
    self.input_text = input_text
    self.string_text = set(' '.join(self.input_text))
    self.char_to_int = {j: i for i,j  in enumerate(self.string_text)}
    self.int_to_char = {v: k for k, v in self.char_to_int.items()}
    self.vocab_size = len(self.string_text)
    self.max_len = len(max(self.input_text, key=len))
    self.input_text_updated = [text + ' '* (maxlen - len(text)) for text in self.input_text]
    self.input_text_to_index = [ list(map(char_to_int.get, text)) for text in  self.input_text_updated]
    self.output_text_to_index = [_input[1:] for _input in self.input_text_to_index]
    self.input_text_to_index_updated = [_input[:-1] for _input in self.input_text_to_index]
  def setup(self,stage):
    #self.train = (F.one_hot(torch.from_numpy(np.array(self.input_text_to_index_updated)),self.vocab_size),torch.Tensor(self.output_text_to_index))
    self.train = (torch.Tensor(self.input_text_to_index_updated),torch.Tensor(self.output_text_to_index))
    
  def train_dataloader(self):
    return DataLoader(self.train, batch_size =3,shuffle = False, num_workers = 1)
class Lit_module(pl.LightningModule):
  def __init__(self,input_size, output_size, hidden_dim, n_layers):
    super().__init__()
    self.input_size = input_size
    self.output_size = output_size
    self.hidden_dim = hidden_dim
    self.n_layers = n_layers
    self.rnn = nn.RNN(self.input_size, self.hidden_dim, self.n_layers, batch_first=True)
    self.fc = nn.Linear(self.hidden_dim, self.output_size)
    self.truncated_bptt_steps = 3

  def init_hidden(self, batch_size):
        hidden = torch.zeros(self.n_layers, batch_size, self.hidden_dim)
        return hidden
  
#   def forward(self, x):
#     batch_size = x.size(0)
#     #Initializing hidden state for first input using method defined below
#     hidden = self.init_hidden(batch_size)
#     # Passing in the input and hidden state into the model and obtaining outputs
#     out, hidden = self.rnn(x, hidden)
#     # Reshaping the outputs such that it can be fit into the fully connected layer
#     out = out.contiguous().view(-1, self.hidden_dim)
#     out = self.fc(out)
#     return out,hidden
  def loss_fn(self,logits,y):
    loss = nn.CrossEntropyLoss()
    return loss(logits,y)
  def training_step(self,train_batch,idx,hiddens = torch.zeros(1, 3, 12) ):
    X,y = train_batch
    X = F.one_hot(X.to(torch.int64),17)
    batch_size = X.size(0)
    #hidden = self.init_hidden(batch_size)
    out, hiddens = self.rnn(X, hiddens)
    #output,hidden = self.forward(X)
    out = out.contiguous().view(-1, self.hidden_dim)
    out = self.fc(out)
    train_loss = criterion(output, y.view(-1).long())
    self.log('train_loss',train_loss)
    return {
            "loss": train_loss,
            "hiddens": hiddens
        }

  def configure_optimizers(self):
    return torch.optim.Adam(model.parameters(), lr= 0.01)

I know it's wrong in several ways. Can someone help me with this?

Here is the error
[---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<command-3372923181664768> in <module>
      4 mlflow.pytorch.autolog()
      5 with mlflow.start_run(experiment_id = 121032840459402, run_name ='densnet') as run:
----> 6     trainer.fit(model, data_module)

/databricks/python/lib/python3.8/site-packages/mlflow/utils/autologging_utils/__init__.py in safe_patch_function(*args, **kwargs)
   1213                         patch_function.call(call_original, *args, **kwargs)
   1214                     else:
-> 1215                         patch_function(call_original, *args, **kwargs)
   1216 
   1217                     try_log_autologging_event(

/databricks/python/lib/python3.8/site-packages/mlflow/utils/autologging_utils/__init__.py in patch_with_managed_run(original, *args, **kwargs)
    987 
    988             try:
--> 989                 result = patch_function(original, *args, **kwargs)
    990             except (Exception, KeyboardInterrupt):
    991                 # In addition to standard Python exceptions, handle keyboard interrupts to ensure

/databricks/python/lib/python3.8/site-packages/mlflow/pytorch/_pytorch_autolog.py in fit(original, self, *args, **kwargs)
    294         Patching trainer.fit method to add autolog class into callback
    295         """
--> 296         return _run_and_log_function(self, original, args, kwargs)
    297 
    298     return fit

/databricks/python/lib/python3.8/site-packages/mlflow/pytorch/_pytorch_autolog.py in _run_and_log_function(self, original, args, kwargs)
    286         if not any(isinstance(callbacks, __MLflowPLCallback) for callbacks in self.callbacks):
    287             self.callbacks += [__MLflowPLCallback()]
--> 288         result = original(self, *args, **kwargs)
    289 
    290         return result

/databricks/python/lib/python3.8/site-packages/mlflow/utils/autologging_utils/__init__.py in call_original(*og_args, **og_kwargs)
   1169                                 disable_warnings=False, reroute_warnings=False,
   1170                             ):
-> 1171                                 original_result = original(*og_args, **og_kwargs)
   1172 
   1173                             try_log_autologging_event(

/databricks/python/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py in fit(self, model, train_dataloader, val_dataloaders, datamodule)
    458         )
    459 
--> 460         self._run(model)
    461 
    462         assert self.state.stopped

/databricks/python/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py in _run(self, model)
    756 
    757         # dispatch `start_training` or `start_evaluating` or `start_predicting`
--> 758         self.dispatch()
    759 
    760         # plugin will finalized fitting (e.g. ddp_spawn will load trained model)

/databricks/python/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py in dispatch(self)
    797             self.accelerator.start_predicting(self)
    798         else:
--> 799             self.accelerator.start_training(self)
    800 
    801     def run_stage(self):

/databricks/python/lib/python3.8/site-packages/pytorch_lightning/accelerators/accelerator.py in start_training(self, trainer)
     94 
     95     def start_training(self, trainer: 'pl.Trainer') -> None:
---> 96         self.training_type_plugin.start_training(trainer)
     97 
     98     def start_evaluating(self, trainer: 'pl.Trainer') -> None:

/databricks/python/lib/python3.8/site-packages/pytorch_lightning/plugins/training_type/training_type_plugin.py in start_training(self, trainer)
    142     def start_training(self, trainer: 'pl.Trainer') -> None:
    143         # double dispatch to initiate the training loop
--> 144         self._results = trainer.run_stage()
    145 
    146     def start_evaluating(self, trainer: 'pl.Trainer') -> None:

/databricks/python/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py in run_stage(self)
    807         if self.predicting:
    808             return self.run_predict()
--> 809         return self.run_train()
    810 
    811     def _pre_training_routine(self):

/databricks/python/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py in run_train(self)
    869                 with self.profiler.profile("run_training_epoch"):
    870                     # run train epoch
--> 871                     self.train_loop.run_training_epoch()
    872 
    873                 if self.max_steps and self.max_steps <= self.global_step:

/databricks/python/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py in run_training_epoch(self)
    497             # ------------------------------------
    498             with self.trainer.profiler.profile("run_training_batch"):
--> 499                 batch_output = self.run_training_batch(batch, batch_idx, dataloader_idx)
    500 
    501             # when returning -1 from train_step, we end epoch early

/databricks/python/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py in run_training_batch(self, batch, batch_idx, dataloader_idx)
    736 
    737                         # optimizer step
--> 738                         self.optimizer_step(optimizer, opt_idx, batch_idx, train_step_and_backward_closure)
    739                         if len(self.trainer.optimizers) > 1:
    740                             # revert back to previous state

/databricks/python/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py in optimizer_step(self, optimizer, opt_idx, batch_idx, train_step_and_backward_closure)
    432 
    433         # model hook
--> 434         model_ref.optimizer_step(
    435             self.trainer.current_epoch,
    436             batch_idx,

/databricks/python/lib/python3.8/site-packages/pytorch_lightning/core/lightning.py in optimizer_step(self, epoch, batch_idx, optimizer, optimizer_idx, optimizer_closure, on_tpu, using_native_amp, using_lbfgs)
   1401 
   1402         """
-> 1403         optimizer.step(closure=optimizer_closure)
   1404 
   1405     def optimizer_zero_grad(self, epoch: int, batch_idx: int, optimizer: Optimizer, optimizer_idx: int):

/databricks/python/lib/python3.8/site-packages/pytorch_lightning/core/optimizer.py in step(self, closure, *args, **kwargs)
    212             profiler_name = f"optimizer_step_and_closure_{self._optimizer_idx}"
    213 
--> 214         self.__optimizer_step(*args, closure=closure, profiler_name=profiler_name, **kwargs)
    215         self._total_optimizer_step_calls += 1
    216 

/databricks/python/lib/python3.8/site-packages/pytorch_lightning/core/optimizer.py in __optimizer_step(self, closure, profiler_name, **kwargs)
    132 
    133         with trainer.profiler.profile(profiler_name):
--> 134             trainer.accelerator.optimizer_step(optimizer, self._optimizer_idx, lambda_closure=closure, **kwargs)
    135 
    136     def step(self, *args, closure: Optional[Callable] = None, **kwargs):

/databricks/python/lib/python3.8/site-packages/pytorch_lightning/accelerators/accelerator.py in optimizer_step(self, optimizer, opt_idx, lambda_closure, **kwargs)
    327         )
    328         if make_optimizer_step:
--> 329             self.run_optimizer_step(optimizer, opt_idx, lambda_closure, **kwargs)
    330         self.precision_plugin.post_optimizer_step(optimizer, opt_idx)
    331         self.training_type_plugin.post_optimizer_step(optimizer, opt_idx, **kwargs)

/databricks/python/lib/python3.8/site-packages/pytorch_lightning/accelerators/accelerator.py in run_optimizer_step(self, optimizer, optimizer_idx, lambda_closure, **kwargs)
    334         self, optimizer: Optimizer, optimizer_idx: int, lambda_closure: Callable, **kwargs: Any
    335     ) -> None:
--> 336         self.training_type_plugin.optimizer_step(optimizer, lambda_closure=lambda_closure, **kwargs)
    337 
    338     def optimizer_zero_grad(self, current_epoch: int, batch_idx: int, optimizer: Optimizer, opt_idx: int) -> None:

/databricks/python/lib/python3.8/site-packages/pytorch_lightning/plugins/training_type/training_type_plugin.py in optimizer_step(self, optimizer, lambda_closure, **kwargs)
    191 
    192     def optimizer_step(self, optimizer: torch.optim.Optimizer, lambda_closure: Callable, **kwargs):
--> 193         optimizer.step(closure=lambda_closure, **kwargs)
    194 
    195     @property

/databricks/python/lib/python3.8/site-packages/torch/optim/optimizer.py in wrapper(*args, **kwargs)
     87                 profile_name = "Optimizer.step#{}.step".format(obj.__class__.__name__)
     88                 with torch.autograd.profiler.record_function(profile_name):
---> 89                     return func(*args, **kwargs)
     90             return wrapper
     91 

/databricks/python/lib/python3.8/site-packages/torch/autograd/grad_mode.py in decorate_context(*args, **kwargs)
     25         def decorate_context(*args, **kwargs):
     26             with self.__class__():
---> 27                 return func(*args, **kwargs)
     28         return cast(F, decorate_context)
     29 

/databricks/python/lib/python3.8/site-packages/torch/optim/adam.py in step(self, closure)
     64         if closure is not None:
     65             with torch.enable_grad():
---> 66                 loss = closure()
     67 
     68         for group in self.param_groups:

/databricks/python/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py in train_step_and_backward_closure()
    730 
    731                         def train_step_and_backward_closure():
--> 732                             result = self.training_step_and_backward(
    733                                 split_batch, batch_idx, opt_idx, optimizer, self.trainer.hiddens
    734                             )

/databricks/python/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py in training_step_and_backward(self, split_batch, batch_idx, opt_idx, optimizer, hiddens)
    821         with self.trainer.profiler.profile("training_step_and_backward"):
    822             # lightning module hook
--> 823             result = self.training_step(split_batch, batch_idx, opt_idx, hiddens)
    824             self._curr_step_result = result
    825 

/databricks/python/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py in training_step(self, split_batch, batch_idx, opt_idx, hiddens)
    288             model_ref._results = Result()
    289             with self.trainer.profiler.profile("training_step"):
--> 290                 training_step_output = self.trainer.accelerator.training_step(args)
    291                 self.trainer.accelerator.post_training_step()
    292 

/databricks/python/lib/python3.8/site-packages/pytorch_lightning/accelerators/accelerator.py in training_step(self, args)
    202 
    203         with self.precision_plugin.train_step_context(), self.training_type_plugin.train_step_context():
--> 204             return self.training_type_plugin.training_step(*args)
    205 
    206     def post_training_step(self) -> None:

/databricks/python/lib/python3.8/site-packages/pytorch_lightning/plugins/training_type/training_type_plugin.py in training_step(self, *args, **kwargs)
    153 
    154     def training_step(self, *args, **kwargs):
--> 155         return self.lightning_module.training_step(*args, **kwargs)
    156 
    157     def post_training_step(self):

<command-4406334066418014> in training_step(self, train_batch, idx, hiddens)
     32     batch_size = X.size(0)
     33     #hidden = self.init_hidden(batch_size)
---> 34     out, hiddens = self.rnn(X, hiddens)
     35     #output,hidden = self.forward(X)
     36     out = out.contiguous().view(-1, self.hidden_dim)

/databricks/python/lib/python3.8/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    887             result = self._slow_forward(*input, **kwargs)
    888         else:
--> 889             result = self.forward(*input, **kwargs)
    890         for hook in itertools.chain(
    891                 _global_forward_hooks.values(),

/databricks/python/lib/python3.8/site-packages/torch/nn/modules/rnn.py in forward(self, input, hx)
    263         _impl = _rnn_impls[self.mode]
    264         if batch_sizes is None:
--> 265             result = _impl(input, hx, self._flat_weights, self.bias, self.num_layers,
    266                            self.dropout, self.training, self.bidirectional, self.batch_first)
    267         else:

RuntimeError: expected scalar type Float but found Long

[hihunjin]

change the following line :

out, hiddens = self.rnn(X, hiddens)

to

out, hiddens = self.rnn(torch.FloatTensor(X), hiddens)

[sushmit86]

This is not woking

[hihunjin]

out, hiddens = self.rnn(torch.FloatTensor(X), torch.FloatTensor(hiddens))

maybe this?

[sushmit86]

TypeError: expected CPU (got CUDA)

[lillekemiker]

When you create the initial hidden tensor, you need to make sure that it is put on the right device (and you can also set the correct dtype then). Something like hiddens = torch.zeros(1, 3, 12, dtype=torch.float, device=self.device) and then you also don't need the torch.FloatTensor bit unless the rnn for some reason returns the wrong tensor type.

[hihunjin]

Tell me how you defined the trainer.