1
2
3
4
5

{
  "instruction": "把下面的句子翻译成英文",
  "input": "今天天气真好",
  "output": "The weather is really nice today."
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58

from unsloth import FastLanguageModel
from trl import SFTTrainer, SFTConfig
from datasets import load_dataset

# ① 加载模型（4bit 量化，省显存）
model, tokenizer = FastLanguageModel.from_pretrained(
    model_name="unsloth/Qwen2.5-7B",
    max_seq_length=2048,
    load_in_4bit=True,       # QLoRA: 4bit 量化
    dtype=None,               # 自动检测
)

# ② 添加 LoRA adapter
model = FastLanguageModel.get_peft_model(
    model,
    r=16,                     # LoRA 秩，越大容量越大
    lora_alpha=16,            # 缩放因子
    lora_dropout=0,           # Unsloth 优化下设 0 即可
    target_modules=["q_proj", "k_proj", "v_proj", 
                    "o_proj", "gate_proj", "up_proj", "down_proj"],
)

# ③ 准备数据
dataset = load_dataset("json", data_files="sft_data.jsonl")

def format_prompt(example):
    return f"""<|im_start|>user
{example['instruction']}
{example['input']}<|im_end|>
<|im_start|>assistant
{example['output']}<|im_end|>"""

# ④ 训练
trainer = SFTTrainer(
    model=model,
    tokenizer=tokenizer,
    train_dataset=dataset["train"],
    args=SFTConfig(
        per_device_train_batch_size=4,
        gradient_accumulation_steps=4,  # 有效 batch = 4×4 = 16
        num_train_epochs=3,
        learning_rate=2e-4,
        warmup_steps=10,
        fp16=True,                      # 混合精度
        logging_steps=10,
        output_dir="./sft_output",
        seed=42,
    ),
)
trainer.train()

# ⑤ 保存
model.save_pretrained("./sft_model")
tokenizer.save_pretrained("./sft_model")

# ⑥ 导出 GGUF（可选，供 Ollama/llama.cpp 使用）
model.save_pretrained_gguf("./sft_gguf", tokenizer, 
                            quantization_method="q4_k_m")

1
2
3
4
5

{
  "prompt": "请用一句话解释什么是机器学习",
  "chosen": "机器学习是让计算机从数据中自动学习规律并做出预测的技术。",
  "rejected": "机器学习就是AI，就是人工智能，很厉害的技术。"
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38

from unsloth import FastLanguageModel, PatchDPOTrainer
from trl import DPOTrainer, DPOConfig

# 重要：Unsloth 的 DPO 优化补丁
PatchDPOTrainer()

# ① 加载 SFT 后的模型（DPO 通常在 SFT 之后做）
model, tokenizer = FastLanguageModel.from_pretrained(
    "./sft_model",
    max_seq_length=2048,
    load_in_4bit=True,
)
model = FastLanguageModel.get_peft_model(model, r=16)

# ② 准备偏好数据
dpo_dataset = load_dataset("json", data_files="dpo_data.jsonl")

# ③ 训练
trainer = DPOTrainer(
    model=model,
    # ref_model=None,  # Unsloth 自动处理 reference model
    tokenizer=tokenizer,
    train_dataset=dpo_dataset["train"],
    args=DPOConfig(
        beta=0.1,              # KL 约束强度。0.1~0.5，越小越激进
        per_device_train_batch_size=2,
        gradient_accumulation_steps=4,
        learning_rate=5e-5,    # 比 SFT 小一个数量级
        num_train_epochs=1,    # DPO 通常 1-2 epochs 就够
        warmup_ratio=0.1,
        fp16=True,
        logging_steps=10,
        output_dir="./dpo_output",
    ),
)
trainer.train()

model.save_pretrained("./dpo_model")

1
2
3
4
5

def math_reward(completion, prompt):
    """数学题：答案对了 +1，错了 -1"""
    predicted = extract_answer(completion)
    ground_truth = get_correct_answer(prompt)
    return 1.0 if predicted == ground_truth else -1.0

1
2
3
4
5
6
7
8

def format_reward(completion, prompt):
    """检查输出是否符合指定格式"""
    score = 0.0
    if "<think>" in completion and "</think>" in completion:
        score += 0.5  # 有思考过程
    if extract_final_answer(completion) is not None:
        score += 0.5  # 有最终答案
    return score

1
2
3
4

def model_reward(completion, prompt):
    """用另一个模型打分"""
    score = reward_model.predict(prompt, completion)
    return score

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63

from unsloth import FastLanguageModel
from trl import GRPOTrainer, GRPOConfig
import re

# ① 加载模型
model, tokenizer = FastLanguageModel.from_pretrained(
    "./sft_model",           # 从 SFT 模型开始
    max_seq_length=2048,
    load_in_4bit=True,
)
model = FastLanguageModel.get_peft_model(model, r=16)

# ② 定义奖励函数
def correctness_reward(completions, prompts, **kwargs):
    """数学正确性奖励"""
    rewards = []
    for comp, prompt in zip(completions, prompts):
        # 提取 <answer>xxx</answer> 中的答案
        match = re.search(r'<answer>(.*?)</answer>', comp)
        if match:
            predicted = match.group(1).strip()
            correct = kwargs.get("answer", [""])[0]
            rewards.append(2.0 if predicted == correct else -1.0)
        else:
            rewards.append(-0.5)  # 格式不对也扣分
    return rewards

def format_reward(completions, **kwargs):
    """格式奖励：鼓励 <think>...</think><answer>...</answer> 结构"""
    rewards = []
    for comp in completions:
        score = 0.0
        if "<think>" in comp and "</think>" in comp:
            score += 0.5
        if "<answer>" in comp and "</answer>" in comp:
            score += 0.5
        rewards.append(score)
    return rewards

# ③ 准备 prompt 数据
# {"prompt": "计算 127 × 38 = ?", "answer": "4826"}
prompts_dataset = load_dataset("json", data_files="math_prompts.jsonl")

# ④ 训练
trainer = GRPOTrainer(
    model=model,
    tokenizer=tokenizer,
    train_dataset=prompts_dataset["train"],
    reward_funcs=[correctness_reward, format_reward],
    args=GRPOConfig(
        num_generations=4,          # 每个 prompt 采样 4 个回答
        max_completion_length=512,  # 最大生成长度
        per_device_train_batch_size=1,
        gradient_accumulation_steps=4,
        learning_rate=1e-5,         # RL 用更小的学习率
        num_train_epochs=1,
        logging_steps=5,
        output_dir="./grpo_output",
    ),
)
trainer.train()

model.save_pretrained("./grpo_model")

1
2
3
4

┌─────────────┐     ┌─────────────┐     ┌──────────────┐
│  Base Model  │────▶│     SFT     │────▶│  DPO / GRPO  │
│  (预训练)    │     │  (学会说话)  │     │  (说得更好)   │
└─────────────┘     └─────────────┘     └──────────────┘

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

你有什么数据？
│
├─ 有 (input, output) 对 ──────────▶ SFT
│
├─ 有 (好回答, 坏回答) 对比 ───────▶ SFT → DPO
│
├─ 有可验证的奖励信号 ─────────────▶ SFT → GRPO
│  （数学答案、代码测试、格式检查）
│
└─ 三者都有 ───────────────────────▶ SFT → DPO → GRPO

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

1. 准备数据 (JSONL)
   ↓
2. 选基座模型 (Qwen2.5 / Llama3 / Gemma)
   ↓
3. SFT 训练 (Unsloth + TRL)
   ↓
4. [可选] DPO / GRPO 对齐
   ↓
5. 导出 (GGUF / safetensors)
   ↓
6. 部署 (Ollama / vLLM / TGI)