|
| 1 | +from unsloth import FastLanguageModel |
| 2 | + |
| 3 | +from typing import Dict |
| 4 | + |
| 5 | +import pytest |
| 6 | +import torch |
| 7 | +from torchao.quantization.qat import FakeQuantizedLinear |
| 8 | +from torchao.quantization.qat.fake_quantizer import ( |
| 9 | + FakeQuantizerBase, |
| 10 | + Float8FakeQuantizer, |
| 11 | + Int4WeightPreshuffledFakeQuantizer, |
| 12 | +) |
| 13 | + |
| 14 | + |
| 15 | +class _CountingFakeQuantizer(torch.nn.Module): |
| 16 | + """ |
| 17 | + Dummy fake quantizer that counts the number of times it has been called. |
| 18 | + """ |
| 19 | + def __init__(self): |
| 20 | + super().__init__() |
| 21 | + self.count = 0 |
| 22 | + |
| 23 | + def forward(self, x: torch.Tensor) -> torch.Tensor: |
| 24 | + self.count += 1 |
| 25 | + return x |
| 26 | + |
| 27 | + |
| 28 | +def _get_model(qat_scheme: str, full_finetuning: bool): |
| 29 | + """ |
| 30 | + Return a 2-tuple of (model, tokenizer), where the model has been configured |
| 31 | + to use QAT. If `full_finetuning` is False, return the PEFT (LoRA) model. |
| 32 | + """ |
| 33 | + model, tokenizer = FastLanguageModel.from_pretrained( |
| 34 | + model_name = "unsloth/Qwen3-1.7B", |
| 35 | + load_in_4bit = False, |
| 36 | + full_finetuning = full_finetuning, |
| 37 | + qat_scheme = qat_scheme if full_finetuning else None, |
| 38 | + ) |
| 39 | + if not full_finetuning: |
| 40 | + model = FastLanguageModel.get_peft_model( |
| 41 | + model, |
| 42 | + qat_scheme = qat_scheme, |
| 43 | + ) |
| 44 | + return model, tokenizer |
| 45 | + |
| 46 | + |
| 47 | +def _test_linear_is_fake_quantized(linear: torch.nn.Linear, qat_scheme: str): |
| 48 | + """ |
| 49 | + Verify that the given linear contains fake quantizers according to the `qat_scheme`. |
| 50 | + """ |
| 51 | + if qat_scheme == "fp8-int4": |
| 52 | + act_fq_class = Float8FakeQuantizer |
| 53 | + weight_fq_class = Int4WeightPreshuffledFakeQuantizer |
| 54 | + min_in_features = 128 |
| 55 | + elif qat_scheme == "fp8-fp8": |
| 56 | + act_fq_class = Float8FakeQuantizer |
| 57 | + weight_fq_class = Float8FakeQuantizer |
| 58 | + min_in_features = -1 |
| 59 | + else: |
| 60 | + raise ValueError(f"Unknown qat_scheme: {qat_scheme}") |
| 61 | + |
| 62 | + # Check base layer activations and weights |
| 63 | + base_layer = getattr(linear, "base_layer", linear) |
| 64 | + if base_layer.in_features >= min_in_features: |
| 65 | + assert isinstance(base_layer, FakeQuantizedLinear) |
| 66 | + assert isinstance(base_layer.activation_fake_quantizer, act_fq_class) |
| 67 | + assert isinstance(base_layer.weight_fake_quantizer, weight_fq_class) |
| 68 | + |
| 69 | + # Check lora A and B (only for full_finetuning=False) |
| 70 | + if hasattr(linear, "lora_A") and hasattr(linear, "lora_B"): |
| 71 | + lora_A = linear.lora_A.default |
| 72 | + lora_B = linear.lora_B.default |
| 73 | + if lora_A.in_features >= min_in_features: |
| 74 | + assert isinstance(lora_A, FakeQuantizedLinear) |
| 75 | + assert isinstance(lora_A.activation_fake_quantizer, act_fq_class) |
| 76 | + assert isinstance(lora_A.weight_fake_quantizer, weight_fq_class) |
| 77 | + if lora_B.in_features >= min_in_features: |
| 78 | + assert isinstance(lora_B, FakeQuantizedLinear) |
| 79 | + assert isinstance(lora_B.activation_fake_quantizer, act_fq_class) |
| 80 | + assert isinstance(lora_B.weight_fake_quantizer, weight_fq_class) |
| 81 | + |
| 82 | + |
| 83 | +def _test_fake_quantizers_are_called( |
| 84 | + model: torch.nn.Module, |
| 85 | + example_inputs: Dict, |
| 86 | + full_finetuning: bool, |
| 87 | +): |
| 88 | + """ |
| 89 | + Verify that the fake quantizers are actually called when the model is called. |
| 90 | + """ |
| 91 | + def _swap_fake_quantizers(model: torch.nn.Module): |
| 92 | + for name, child in model.named_children(): |
| 93 | + if isinstance(child, FakeQuantizerBase): |
| 94 | + setattr(model, name, _CountingFakeQuantizer()) |
| 95 | + |
| 96 | + def _assert_fake_quantizers_are_called(model: torch.nn.Module): |
| 97 | + for name, child in model.named_children(): |
| 98 | + if full_finetuning: |
| 99 | + if isinstance(child, FakeQuantizedLinear): |
| 100 | + assert child.activation_fake_quantizer.count == 1 |
| 101 | + assert child.weight_fake_quantizer.count == 1 |
| 102 | + else: |
| 103 | + # For LoRA, we only fake quantize the input activations once per block: |
| 104 | + # For self_attn, we only fake quantize the q_proj's input activations |
| 105 | + # For mlp, we only fake quantize the gate_proj's input activations |
| 106 | + if name == "self_attn": |
| 107 | + base_layer = child.q_proj.base_layer |
| 108 | + assert hasattr(base_layer, "activation_fake_quantizer") |
| 109 | + assert base_layer.activation_fake_quantizer.count == 1 |
| 110 | + elif name == "mlp": |
| 111 | + base_layer = child.gate_proj.base_layer |
| 112 | + assert hasattr(base_layer, "activation_fake_quantizer") |
| 113 | + assert base_layer.activation_fake_quantizer.count == 1 |
| 114 | + elif isinstance(child, FakeQuantizedLinear): |
| 115 | + # Weight fake quantizers should always be called |
| 116 | + assert child.weight_fake_quantizer.count == 1 |
| 117 | + |
| 118 | + for k, v in example_inputs.items(): |
| 119 | + example_inputs[k] = v.cuda() |
| 120 | + model.apply(_swap_fake_quantizers) |
| 121 | + model(**example_inputs) |
| 122 | + model.apply(_assert_fake_quantizers_are_called) |
| 123 | + |
| 124 | + |
| 125 | +def _test_model_fake_quantize(qat_scheme: bool, full_finetuning: bool): |
| 126 | + """ |
| 127 | + Test that all linear layers in the model are fake quantized according to the `qat_scheme`. |
| 128 | + """ |
| 129 | + model, tokenizer = _get_model(qat_scheme, full_finetuning) |
| 130 | + if full_finetuning: |
| 131 | + model = model.model |
| 132 | + else: |
| 133 | + model = model.base_model.model.model |
| 134 | + for layer in model.layers: |
| 135 | + _test_linear_is_fake_quantized(layer.self_attn.q_proj, qat_scheme) |
| 136 | + _test_linear_is_fake_quantized(layer.self_attn.k_proj, qat_scheme) |
| 137 | + _test_linear_is_fake_quantized(layer.self_attn.v_proj, qat_scheme) |
| 138 | + _test_linear_is_fake_quantized(layer.mlp.gate_proj, qat_scheme) |
| 139 | + _test_linear_is_fake_quantized(layer.mlp.up_proj, qat_scheme) |
| 140 | + _test_linear_is_fake_quantized(layer.mlp.down_proj, qat_scheme) |
| 141 | + inputs = tokenizer("How are you?", return_tensors="pt") |
| 142 | + _test_fake_quantizers_are_called(model, inputs, full_finetuning) |
| 143 | + |
| 144 | + |
| 145 | +# TODO: there are bad interactions across tests right now, need to figure out |
| 146 | +# how to disable model caching before re-enabling this test |
| 147 | +@pytest.mark.parametrize("qat_scheme", ["fp8-int4", "fp8-fp8"]) |
| 148 | +def _test_full_model_fake_quantize(qat_scheme: bool): |
| 149 | + _test_model_fake_quantize(qat_scheme, full_finetuning=True) |
| 150 | + |
| 151 | + |
| 152 | +@pytest.mark.parametrize("qat_scheme", ["fp8-int4", "fp8-fp8"]) |
| 153 | +def test_lora_model_fake_quantize(qat_scheme: bool): |
| 154 | + _test_model_fake_quantize(qat_scheme, full_finetuning=False) |
0 commit comments