python 3.x - Feeding Classifier data from LSTM Autoencoder

Question

Goal:

I have built a LSTM autoencoder for the purpose of feature reduction. My plan is to encode some input and feed it to a classifier in the future. The encoder takes data of the shape [batch_size, timesteps, features_of_timesteps However in the output layer of the encoder portion I am returning just the last hidden state in the form [1, timesteps, features_of_timesteps].

class Encoder(nn.Module):
    def __init__(self, input_size, first_layer, second_layer, n_layers):
        super(Encoder, self).__init__()
        self.n_layers = n_layers
        self.encode = nn.Sequential(nn.LSTM(input_size, first_layer, batch_first=True),
                                    getSequence(),
                                    nn.ReLU(True),
                                    nn.LSTM(first_layer, second_layer),
                                    getLast())
        self.decode = nn.Sequential(nn.LSTM(second_layer, first_layer, batch_first=True),
                                    getSequence(),
                                    nn.ReLU(True),
                                    nn.LSTM(first_layer, input_size),
                                    getSequence())

    def forward(self, x):
        x = x.float()
        x = self.encode(x)
        x = x.repeat(batch_size, 1, 1)
        x = self.decode(x)
        return x

Worry:

I am afraid that the last hidden state of the my second LSTM layer in the encoding portion of the model is summarizing the entire batch along with decreasing the feature dimensionality. This feels wrong because I am trying to reduce a single timeseries into a smaller vector, not an entire batch of timeseries into one vector. Am I correct in my worries?

question from:https://stackoverflow.com/questions/65876808/feeding-classifier-data-from-lstm-autoencoder

Answer 1

There have multiple problem in your code, for simplicity, I just give you one well defined model instead, following code build a LSTM Autoencoder that reconstruct the inputs with shape (batch_size, timesteps, number_of_features_at_each_timesteps):

import torch
from torch import nn

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

class Encoder(nn.Module):
  def __init__(self, seq_len, n_features, embedding_dim=64):
    super(Encoder, self).__init__()

    self.seq_len, self.n_features = seq_len, n_features
    self.embedding_dim, self.hidden_dim = embedding_dim, 2 * embedding_dim

    self.rnn1 = nn.LSTM(
      input_size=n_features,
      hidden_size=self.hidden_dim,
      num_layers=1,
      batch_first=True
    )
    self.rnn2 = nn.LSTM(
      input_size=self.hidden_dim,
      hidden_size=self.embedding_dim,
      num_layers=1,
      batch_first=True
    )

  def forward(self, x):
    x, (_, _) = self.rnn1(x)
    x, (hidden_n, _) = self.rnn2(x)
    return hidden_n

class Decoder(nn.Module):
  def __init__(self, seq_len, input_dim=64, n_features=1):
    super(Decoder, self).__init__()

    self.seq_len, self.input_dim = seq_len, input_dim
    self.hidden_dim, self.n_features = 2 * input_dim, n_features

    self.rnn1 = nn.LSTM(
      input_size=input_dim,
      hidden_size=input_dim,
      num_layers=1,
      batch_first=True
    )
    self.rnn2 = nn.LSTM(
      input_size=input_dim,
      hidden_size=self.hidden_dim,
      num_layers=1,
      batch_first=True
    )
    self.output_layer = nn.Linear(self.hidden_dim, n_features)

  def forward(self, x):
    x = x.repeat(self.seq_len, 1, 1)
    x = x.permute(1, 0, 2)
    x, (hidden_n, cell_n) = self.rnn1(x)
    x, (hidden_n, cell_n) = self.rnn2(x)
    return self.output_layer(x)

class RecurrentAutoencoder(nn.Module):
  def __init__(self, seq_len, n_features, embedding_dim=64):
    super(RecurrentAutoencoder, self).__init__()

    self.encoder = Encoder(seq_len, n_features, embedding_dim).to(device)
    self.decoder = Decoder(seq_len, embedding_dim, n_features).to(device)

  def forward(self, x):
    print("Inputs size:", x.size())
    x = self.encoder(x)
    print("Representation size: ", x.size())
    x = self.decoder(x)
    print("Outputs size: ", x.size())
    return x

batch_n = 5
seq_len = 10
n_features = 3
inputs = torch.randn(batch_n, seq_len, n_features).to(device)

model = RecurrentAutoencoder(seq_len, n_features).to(device)
y = model(inputs)

Outputs:

Inputs size: torch.Size([5, 10, 3])
Representation size:  torch.Size([1, 5, 64])
Outputs size:  torch.Size([5, 10, 3])

Beware the representation (i.e outputs of encoder) have shape (1, batch_size, embedding_dim)