Inital commit

convertinglinear.py

@@ -0,0 +1,29 @@
+import torch
+
+
+class ConvertingLinear(torch.nn.Linear):
+    def __init__(self, in_features, out_features, bias=True, device=None, dtype=None):
+        super().__init__(in_features, out_features, bias, device, dtype)
+
+    def forward(self, input: torch.Tensor):
+        output_dtype = input.dtype
+        output_device = input.device
+        if input.device != self.weight.device:
+            input = input.to(self.weight.device)
+        if input.dtype != self.weight.dtype:
+            input = input.to(self.weight.dtype)
+        output = torch.nn.Linear.forward(self, input)
+        if torch.isnan(output).any() or self.weight.dtype != torch.float32:
+            breakpoint()
+        return output.to(output_device).to(output_dtype)
+
+    @classmethod
+    def fromLinear(cls, in_module: torch.nn.Linear):
+        new_module = torch.nn.utils.skip_init(cls, in_features=in_module.in_features,
+                                              out_features=in_module.out_features,
+                                              bias=in_module.bias is not None,
+                                              device=in_module.weight.device,
+                                              dtype=in_module.weight.dtype)
+        new_module.weight = in_module.weight
+        new_module.bias = in_module.bias
+        return new_module