30 lines
1.3 KiB
Python
30 lines
1.3 KiB
Python
import torch
|
|
|
|
|
|
class ConvertingLinear(torch.nn.Linear):
|
|
def __init__(self, in_features, out_features, bias=True, device=None, dtype=None):
|
|
super().__init__(in_features, out_features, bias, device, dtype)
|
|
|
|
def forward(self, input: torch.Tensor):
|
|
output_dtype = input.dtype
|
|
output_device = input.device
|
|
if input.device != self.weight.device:
|
|
input = input.to(self.weight.device)
|
|
if input.dtype != self.weight.dtype:
|
|
input = input.to(self.weight.dtype)
|
|
output = torch.nn.Linear.forward(self, input)
|
|
if torch.isnan(output).any() or self.weight.dtype != torch.float32:
|
|
breakpoint()
|
|
return output.to(output_device).to(output_dtype)
|
|
|
|
@classmethod
|
|
def fromLinear(cls, in_module: torch.nn.Linear):
|
|
new_module = torch.nn.utils.skip_init(cls, in_features=in_module.in_features,
|
|
out_features=in_module.out_features,
|
|
bias=in_module.bias is not None,
|
|
device=in_module.weight.device,
|
|
dtype=in_module.weight.dtype)
|
|
new_module.weight = in_module.weight
|
|
new_module.bias = in_module.bias
|
|
return new_module
|