fix: correct LoRALinear dtype and remove unused import
- LoRALinear now creates lora_A/lora_B with dtype matching the base linear's weight, preventing a float32/bf16 mismatch at forward time when the generator is loaded in bf16 or fp16. - Remove unused `import math` from train_lora.py. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -41,8 +41,9 @@ class LoRALinear(nn.Module):
|
||||
if linear.bias is not None:
|
||||
linear.bias.requires_grad_(False)
|
||||
|
||||
self.lora_A = nn.Parameter(torch.empty(rank, in_f))
|
||||
self.lora_B = nn.Parameter(torch.zeros(out_f, rank))
|
||||
ref_dtype = linear.weight.dtype
|
||||
self.lora_A = nn.Parameter(torch.empty(rank, in_f, dtype=ref_dtype))
|
||||
self.lora_B = nn.Parameter(torch.zeros(out_f, rank, dtype=ref_dtype))
|
||||
self.scale = alpha / rank
|
||||
|
||||
nn.init.kaiming_uniform_(self.lora_A, a=math.sqrt(5))
|
||||
|
||||
Reference in New Issue
Block a user