BlenderbotForCausalLM(
  (model): BlenderbotDecoderWrapper(
    (decoder): BlenderbotDecoder(
      (embed_tokens): Embedding(8008, 2560, padding_idx=0)
      (embed_positions): BlenderbotLearnedPositionalEmbedding(128, 2560)
      (layers): ModuleList(
        (0-23): 24 x BlenderbotDecoderLayer(
          (self_attn): BlenderbotAttention(
            (k_proj): Linear(in_features=2560, out_features=2560, bias=True)
            (v_proj): Linear(in_features=2560, out_features=2560, bias=True)
            (q_proj): Linear(in_features=2560, out_features=2560, bias=True)
            (out_proj): Linear(in_features=2560, out_features=2560, bias=True)
          )
          (activation_fn): GELUActivation()
          (self_attn_layer_norm): LayerNorm((2560,), eps=1e-05, elementwise_affine=True)
          (encoder_attn): BlenderbotAttention(
            (k_proj): Linear(in_features=2560, out_features=2560, bias=True)
            (v_proj): Linear(in_features=2560, out_features=2560, bias=True)
            (q_proj): Linear(in_features=2560, out_features=2560, bias=True)
            (out_proj): Linear(in_features=2560, out_features=2560, bias=True)
          )
          (encoder_attn_layer_norm): LayerNorm((2560,), eps=1e-05, elementwise_affine=True)
          (fc1): Linear(in_features=2560, out_features=10240, bias=True)
          (fc2): Linear(in_features=10240, out_features=2560, bias=True)
          (final_layer_norm): LayerNorm((2560,), eps=1e-05, elementwise_affine=True)
        )
      )
      (layer_norm): LayerNorm((2560,), eps=1e-05, elementwise_affine=True)
    )
  )
  (lm_head): Linear(in_features=2560, out_features=8008, bias=False)
)