python - Efficient net incompatible sizes for mixed data

Question

I have some code for a mixed model, one that trains on an efficient net and the rest on some external data that I have combined. The following is an example for the model:

def create_model():
# Define parameters
inputShape = (256,256,3)
inputDim = 8
# define MLP network
model = Sequential()
model.add(Dense(8, input_dim=inputDim, activation="relu"))
model.add(Dense(4, activation="relu"))
cnnModel = Sequential()
cnnModel.add(EfficientNetB5(include_top = False, input_shape=inputShape))
cnnModel.add(Flatten())
cnnModel.add(Dense(units = 16, activation='relu'))
cnnModel.add(Dense(units = 4, activation='relu'))
# Concatenate them
fullModel = concatenate([cnnModel.output,model.output])
fullModel = Dense(4, activation="relu")(fullModel)
fullModel = Dense(1, activation="sigmoid")(fullModel)
model = Model(inputs=[cnnModel.input,model.input], outputs=fullModel)
return model

However, when I run this through the fit_generator function I recieve the following error:

batch_size = 16
train_steps = TrainData.shape[0]//batch_size
valid_steps = TrainData.shape[0]//batch_size
model = create_model()
opt = Adam(lr=1e-3, decay=1e-3 / 200)
model.compile(loss="binary_crossentropy", optimizer=opt)
print("[INFO] training model...")
model.fit_generator(
    train_dl,
    epochs=3,
    steps_per_epoch = train_steps
)
model.save("models/final_model")

InvalidArgumentError:  Incompatible shapes: [16,3,256,256] vs. [1,1,1,3]
     [[node model_47/efficientnetb5/normalization_52/sub (defined at <ipython-input-262-76be6a4af4a4>:11) ]] [Op:__inference_train_function_1072272]

I'm unsure where this error is coming from, either in the data loader or in the efficient net. Any ideas?

Edit to include data loader:

def data_generator(image_dir, dataframe, min_max, binary, category, transforms = None, batch_size = 16):
  i = 0
  samples_per_epoch = dataframe.shape[0]
  number_of_batches = samples_per_epoch/batch_size
  while True:
      batch = {'images': [], 'data': [], 'labels': []}  # use a dict for multiple inputs
      # Randomly sample images in dataframe
      idx = i
      img_path = f"{image_dir}/{dataframe.iloc[idx]['image_name']}.jpg"
      img = Image.open(img_path)
      if transforms:
          img = transforms(**{"image": np.array(img)})["image"]
      img = np.asarray( img, dtype="int32" )
      # make data into tensors
      dataframe2 = dataframe.iloc[idx]
      data_cont = min_max.transform(np.array(dataframe2['age_approx']).reshape(1, -1))
      data_bina = binary.transform(dataframe2['sex'])
      data_cate = category.transform(dataframe2['anatom_site_general_challenge'])
      data_total = np.concatenate((data_cont, data_bina, data_cate), axis = 1)
      label = dataframe2['target']
      batch['images'].append(img)
      batch['data'].append(data_total)
      batch['labels'].append(label)
      batch['images'] = np.array(batch['images'])  # convert each list to array
      batch['data'] = np.array(batch_x['data'])
      batch['labels'] = np.array(batch['labels'])
      i += 1
      if counter >= number_of_batches:
        counter = 0
      yield [batch['images'], batch['data']], batch['labels']

def get_data(train_df, valid_df, train_tfms, test_tfms, batch_size, min_max, binary, category):
  train_dl = data_generator(image_dir='train/', dataframe = train_df, batch_size = batch_size, min_max = min_max, binary = binary, category = category, transforms = train_tfms)
  valid_dl = data_generator(image_dir='train/', dataframe = valid_df, batch_size = batch_size*2, min_max = min_max, binary = binary, category = category, transforms = test_tfms)
  return train_dl, valid_dl

I seem to have the same issue when I just used the images and the efficient net. It seems like using the Keras inbuilt image data loader functions is the only way I can get it to work (with just images).

question from:https://stackoverflow.com/questions/65713216/efficient-net-incompatible-sizes-for-mixed-data

Answer 1

Waitting for answers