diff --git a/config.json b/config.json index c6bb9d0..519c9b3 100644 --- a/config.json +++ b/config.json @@ -40,17 +40,12 @@ "use_exllama": true }, "rms_norm_eps": 1e-06, - "rope_scaling": { - "factor": 4.0, - "original_max_position_embeddings": 32768, - "type": "yarn" - }, "rope_theta": 1000000.0, - "sliding_window": 32768, + "sliding_window": 131072, "tie_word_embeddings": false, "torch_dtype": "float16", "transformers_version": "4.39.3", "use_cache": true, "use_sliding_window": false, "vocab_size": 152064 -} \ No newline at end of file +}