Update README.md

README.md

@@ -6,11 +6,12 @@ tags:
 - llama-factory
 - lora
 datasets:
+  - Snit/french-conversation
   - Nekochu/novel17_train_alpaca_format
   - bofenghuang/vigogne
-  - jpacifico/French-Alpaca-dataset-Instruct-110K
   - MaziyarPanahi/french_instruct_human_sharegpt
-  - Snit/french-conversation
+  - jpacifico/French-Alpaca-dataset-Instruct-110K
+  - jpacifico/french-orca-dpo-pairs-revised
   
 language:
   - fr
@@ -33,8 +34,91 @@ Stage B: Continued **S**upervised **F**ine-**T**uning, QA
 set CUDA_VISIBLE_DEVICES=0 && llamafactory-cli train --stage sft --do_train True --model_name_or_path NousResearch/Meta-Llama-3.1-8B-Instruct --preprocessing_num_workers 16 --finetuning_type lora --template alpaca --rope_scaling linear --flash_attn fa2 --dataset_dir data --dataset Acquiesce_french_vigogne,novel17_train --cutoff_len 8192 --learning_rate 5e-05 --num_train_epochs 3.0 --max_samples 10000000 --per_device_train_batch_size 1 --gradient_accumulation_steps 1 --lr_scheduler_type cosine --max_grad_norm 1.0 --logging_steps 10 --save_steps 1000 --warmup_steps 0 --neftune_noise_alpha 5 --optim adamw_8bit --packing True --report_to none --output_dir saves\LLaMA3.1-8B-Chat\lora\QLoRA_french_sft --bf16 True --plot_loss True --ddp_timeout 180000000 --adapter_name_or_path saves\LLaMA3.1-8B-Chat\lora\QLoRA_french_pt --quantization_bit 4 --quantization_method bitsandbytes --lora_rank 32 --lora_alpha 64 --lora_dropout 0.15 --lora_target all
 ```
 
-Dataset convert to Alpaca: [Acquiesce_french_vigogne](https://huggingface.co/datasets/Nekochu/Luminia-mixture/tree/split-v2/General/French),french-raw-pt
+Stage C: Continued **D**irect **P**reference **O**ptimization
+
+<details>
+  <summary>Config</summary>
+
+`llama3_lora_dpo.yaml`
+```yaml
+### model:    
+model_name_or_path: NousResearch/Meta-Llama-3.1-8B-Instruct
+quantization_bit: 4
+use_adam_mini: true
+adapter_name_or_path: saves\LLaMA3.1-8B-Chat\lora\QLoRA_french_sft 
 
-<!-- DPO¿: https://huggingface.co/datasets/jpacifico/french-orca-dpo-pairs-revised -->
+### method
+stage: dpo
+do_train: true
+finetuning_type: lora
+lora_target: all
+pref_beta: 0.1
+pref_loss: sigmoid  # choices: [sigmoid (dpo), orpo, simpo]
+
+### dataset
+dataset: french_orca_rlhf-revised
+template: llama3
+cutoff_len: 4096
+max_samples: 10000
+overwrite_cache: true
+preprocessing_num_workers: 16
+
+### output
+output_dir: saves/LLaMA3.1-8B-Chat/lora/QLoRA_french_dpo
+logging_steps: 10
+save_steps: 1000
+plot_loss: true
+overwrite_output_dir: true
+
+### train
+per_device_train_batch_size: 1
+gradient_accumulation_steps: 1
+learning_rate: 5.0e-6
+num_train_epochs: 1.0
+lr_scheduler_type: cosine
+warmup_ratio: 0.1
+bf16: true
+ddp_timeout: 180000000
+flash_attn: fa2
+optim: paged_adamw_8bit
+```
+
+`dataset_info.json`:
+```json
+  "french_orca_rlhf-revised": {
+    "hf_hub_url": "jpacifico/french-orca-dpo-pairs-revised",
+    "ranking": true,
+    "columns": {
+      "prompt": "question",
+      "chosen": "chosen",
+      "rejected": "rejected",
+      "system": "system"
+    }
+  },
+  "novel17_train": {
+    "hf_hub_url": "Nekochu/novel17_train_alpaca_format",
+    "formatting": "alpaca"
+  },
+  "Acquiesce_french_vigogne": {
+    "file_name": "Acquiesce_french_vigogne.json",
+    "formatting": "alpaca",
+    "columns": {
+      "prompt": "instruction",
+      "query": "input",
+      "response": "output",
+      "system": "system",
+      "history": "history"
+    }
+  },
+  "french-raw-pt": {
+    "file_name": "french-raw-pt.json",
+    "columns": {
+      "prompt": "text"
+    }
+  },
+```
+</details>
+
+Dataset convert to Alpaca: [Acquiesce_french_vigogne](https://huggingface.co/datasets/Nekochu/Luminia-mixture/tree/split-v2/General/French),french-raw-pt
 
 </details>