Model Finetune Params and dependencies

Gemma2_2B/hyperparams.yaml

@@ -1,7 +1,7 @@
 model_name: "google/gemma-2-2b-it"
 new_model_name: "gemma-2-2b-ft"
 
-lora_r: 4
+lora_r: 64
 lora_alpha: 16
 lora_dropout: 0.1
 
@@ -11,7 +11,7 @@ bnb_4bit_quant_type: "nf4"
 use_nested_quant: False
 
 output_dir: "./results"
-num_train_epochs: 1
+num_train_epochs: 2
 fp16: False
 bf16: False
 per_device_train_batch_size: 2
@@ -19,7 +19,7 @@ per_device_eval_batch_size: 2
 gradient_accumulation_steps: 1
 gradient_checkpointing: True
 max_grad_norm: 0.3
-learning_rate: 2e-4
+learning_rate: 2e-3
 weight_decay: 0.001
 optimizer: "paged_adamw_32bit"
 lr_scheduler_type: "constant"
@@ -29,6 +29,6 @@ group_by_length: True
 save_steps: 25
 logging_steps: 25
 
-max_seq_length: 40
+max_seq_length: 128
 packing: True
 device_map: "auto"

pyproject.toml

@@ -1,5 +1,5 @@
 [project]
-name = "learninguv"
+name = "TADBot"
 version = "0.1.0"
 description = "Add your description here"
 readme = "README.md"
@@ -26,6 +26,8 @@ dependencies = [
     "ipykernel>=6.29.5",
     "ipywidgets>=8.1.5",
     "pyyaml>=6.0.2",
+    "torch-tb-profiler>=0.4.3",
+    "tensorflow>=2.18.0",
 ]
 
 [tool.uv.sources]