{
  "name": "rotation",
  "backbone": "model",
  "model_decoder_layers": "model.layers",
  "v_proj": "self_attn.v_proj",
  "o_proj": "self_attn.o_proj",
  "self_attn": "self_attn",
  "mlp": "mlp",
  "r1": true,
  "r2": true,
  "r3": false,
  "r4": false,
  "trainable": true,
  "online_r1_rotation": true,
  "online_config": {
    "shared_parallel": true
  },
  "rotation_size": 128,
  "scaling_layers": {
    "first_layer": [
      {
        "prev_modules": [
          "model.embed_tokens"
        ],
        "norm_module": "model.layers.layer_id.input_layernorm",
        "next_modules": [
          "model.layers.layer_id.self_attn.q_proj",
          "model.layers.layer_id.self_attn.k_proj",
          "model.layers.layer_id.self_attn.v_proj"
        ]
      },
      {
        "prev_modules": [
          "model.layers.layer_id.self_attn.o_proj"
        ],
        "norm_module": "model.layers.layer_id.post_attention_layernorm",
        "next_modules": [
          "model.layers.layer_id.mlp.up_proj",
          "model.layers.layer_id.mlp.gate_proj"
        ]
      }
    ],
    "middle_layers": [
      {
        "prev_modules": [
          "model.layers.pre_layer_id.mlp.down_proj"
        ],
        "norm_module": "model.layers.layer_id.input_layernorm",
        "next_modules": [
          "model.layers.layer_id.self_attn.q_proj",
          "model.layers.layer_id.self_attn.k_proj",
          "model.layers.layer_id.self_attn.v_proj"
        ]
      },
      {
        "prev_modules": [
          "model.layers.layer_id.self_attn.o_proj"
        ],
        "norm_module": "model.layers.layer_id.post_attention_layernorm",
        "next_modules": [
          "model.layers.layer_id.mlp.up_proj",
          "model.layers.layer_id.mlp.gate_proj"
        ]
      }
    ],
    "last_layer": [
      {
        "prev_modules": [
          "model.layers.layer_id.mlp.down_proj"
        ],
        "norm_module": "model.norm",
        "next_modules": [
          "lm_head"
        ]
      }
    ]
  }
}
