Upload 8_entities_top_span_self_ensemble_no_weight_20's state dict

8_entities_top_span_self_ensemble_no_weight_20/8_entities_top_span_self_ensemble_no_weight_20.py

@@ -467,227 +467,6 @@ class SpanErrorAnalyzer:
             "details": detailed_errors
         }
 
-# %% [code]
-class DataParallelProxy(nn.DataParallel):
-
-    def __getattr__(self, name):
-        try:
-            return super().__getattr__(name)
-
-        except AttributeError:
-
-            attr = getattr(self.module, name)
-
-            if callable(attr):
-
-                def wrapper(*args, **kwargs):
-                    return self._parallel_apply_method(
-                        name,
-                        *args,
-                        **kwargs
-                    )
-
-                return wrapper
-
-            return attr
-
-    # =========================================================
-    # parallel custom method
-    # =========================================================
-
-    def _parallel_apply_method(self, method_name, *inputs, **kwargs):
-
-        if not self.device_ids:
-            return getattr(self.module, method_name)(*inputs, **kwargs)
-
-        inputs_scattered, kwargs_scattered = self.scatter(
-            inputs,
-            kwargs,
-            self.device_ids
-        )
-
-        replicas = self.replicate(
-            self.module,
-            self.device_ids[:len(inputs_scattered)]
-        )
-
-        outputs = self.parallel_apply(
-            [getattr(replica, method_name) for replica in replicas],
-            inputs_scattered,
-            kwargs_scattered
-        )
-
-        return self._custom_gather(
-            outputs,
-            self.output_device
-        )
-
-    # =========================================================
-    # OVERRIDE FORWARD GATHER
-    # =========================================================
-
-    def gather(self, outputs, output_device):
-
-        return self._custom_gather(
-            outputs,
-            output_device
-        )
-
-    # =========================================================
-    # recursive gather
-    # =========================================================
-
-    def _custom_gather(self, outputs, output_device):
-
-        first = outputs[0]
-
-        # =====================================================
-        # tensor
-        # =====================================================
-
-        if torch.is_tensor(first):
-
-            return self._gather_tensor(
-                outputs,
-                output_device
-            )
-
-        # =====================================================
-        # tuple
-        # =====================================================
-
-        if isinstance(first, tuple):
-
-            return tuple(
-                self._custom_gather(
-                    list(items),
-                    output_device
-                )
-                for items in zip(*outputs)
-            )
-
-        # =====================================================
-        # list
-        # =====================================================
-
-        if isinstance(first, list):
-
-            # list[tensor]
-            if len(first) > 0 and torch.is_tensor(first[0]):
-
-                return self._gather_tensor_list(
-                    outputs,
-                    output_device
-                )
-
-            merged = []
-
-            for out in outputs:
-                merged.extend(out)
-
-            return merged
-
-        # =====================================================
-        # dict
-        # =====================================================
-
-        if isinstance(first, dict):
-
-            return {
-                k: self._custom_gather(
-                    [o[k] for o in outputs],
-                    output_device
-                )
-                for k in first.keys()
-            }
-
-        # =====================================================
-        # fallback
-        # =====================================================
-
-        return outputs
-
-    # =========================================================
-    # gather tensor with auto pad
-    # =========================================================
-
-    def _gather_tensor(self, tensors, output_device):
-
-        # move same device first
-        tensors = [
-            t.to(output_device)
-            for t in tensors
-        ]
-
-        # =====================================================
-        # fast path
-        # =====================================================
-
-        try:
-            return torch.cat(tensors, dim=0)
-
-        except RuntimeError:
-            pass
-
-        # =====================================================
-        # auto max shape
-        # =====================================================
-
-        max_shape = list(tensors[0].shape)
-
-        for t in tensors[1:]:
-
-            for d in range(len(max_shape)):
-
-                max_shape[d] = max(
-                    max_shape[d],
-                    t.shape[d]
-                )
-
-        # =====================================================
-        # pad tensors
-        # =====================================================
-
-        padded = []
-
-        for t in tensors:
-
-            pad = []
-
-            # reverse order for F.pad
-            for d in reversed(range(len(max_shape))):
-
-                # never pad batch dim
-                if d == 0:
-                    pad.extend([0, 0])
-                    continue
-
-                diff = max_shape[d] - t.shape[d]
-
-                pad.extend([0, diff])
-
-            t = F.pad(t, pad)
-
-            padded.append(t)
-
-        return torch.cat(padded, dim=0)
-
-    # =========================================================
-    # list[tensor]
-    # =========================================================
-
-    def _gather_tensor_list(self, outputs, output_device):
-
-        merged = []
-
-        for out in outputs:
-            merged.extend(out)
-
-        return self._gather_tensor(
-            merged,
-            output_device
-        )
-
 # %% [code]
 ## Viết cấu trúc model vào đây
 def get_span_reprs(hidden, spans):
@@ -885,7 +664,7 @@ class IEModel(nn.Module):
         return start_logits, end_logits, logits, spans
 
 def test_model():
-    model = DataParallelProxy(IEModel(backbone_model_name, 7, 10, 100, 2)).to(device)
+    model = nn.DataParallel(IEModel(backbone_model_name, 7, 10, 100, 0)).to(device)
     model.eval()
     total_params = sum(p.numel() for p in model.parameters())
     print(f"Total params: {total_params:,}")
@@ -984,18 +763,41 @@ def fmt(x):
     return x
     
 class ModelEmaV3Proxy(ModelEmaV3):
-
     def __getattr__(self, name):
-
         try:
             return super().__getattr__(name)
-
         except AttributeError:
+            return getattr(self.module, name)
+            
+class DataParallelProxy(nn.DataParallel):
+    def __getattr__(self, name):
+        try:
+            return super().__getattr__(name)
+        except AttributeError:
+            attr = getattr(self.module, name)
 
-            # tránh recursion
-            module = object.__getattribute__(self, "module")
+            if callable(attr):
+                def wrapper(*args, **kwargs):
+                    return self._parallel_apply_method(name, *args, **kwargs)
+                return wrapper
 
-            return getattr(module, name)
+            return attr
+
+    def _parallel_apply_method(self, method_name, *inputs, **kwargs):
+        if not self.device_ids:
+            return getattr(self.module, method_name)(*inputs, **kwargs)
+
+        inputs_scattered, kwargs_scattered = self.scatter(inputs, kwargs, self.device_ids)
+
+        replicas = self.replicate(self.module, self.device_ids)
+
+        outputs = self.parallel_apply(
+            [getattr(replica, method_name) for replica in replicas],
+            inputs_scattered,
+            kwargs_scattered
+        )
+
+        return self.gather(outputs, self.output_device)
 
 def align(
     all_spans,   # (B, N, 2)
@@ -1277,7 +1079,7 @@ class Trainer:
                 self.ema_net = ModelEmaV3Proxy(network, self.ema_decay, device=self.device)
                 
             try:
-                teaching_rate = math.cos(math.pi / 2 * epoch / epochs)
+                teaching_rate = math.cos(math.pi / 2 * (epoch - 2) / (epochs - 2)) if epoch - 2 > 0 else 1.0
                 train_loss_epoch, train_loss_epoch_dict = self._train_epoch(network, train_loader, optimizer, scheduler, loss_fn, teaching_rate)
                 logging_dict = {'lr': [group['lr'] for group in optimizer.param_groups], 'train_loss': train_loss_epoch}
                 logging_dict.update(train_loss_epoch_dict)
@@ -1455,11 +1257,17 @@ class Trainer:
         start_labels = batch['start_labels'].to(self.device)
         end_labels = batch['end_labels'].to(self.device)
 
+        hidden_states, attention_mask = network.encode(input_ids, attention_mask)
+        start_logits, end_logits = network.get_token_logits(hidden_states)
+
         choice = random.random()
         if choice < teaching_rate:
-            start_logits, end_logits, logits, pred_spans = network(input_ids, attention_mask, all_spans)
+            pred_spans = all_spans
         else:
-            start_logits, end_logits, logits, pred_spans = network(input_ids, attention_mask)
+            pred_spans = filter_spans(start_logits, end_logits, attention_mask, network.max_span_len, network.topk_spans, network.keep_neighbor)
+            
+        span_reprs = get_span_reprs(hidden_states, pred_spans)
+        logits = network.get_logits(span_reprs)
 
         align_labels = align(all_spans, pred_spans, all_labels, -100)
         align_weights = align(all_spans, pred_spans, all_weights, 0)
@@ -1481,7 +1289,11 @@ class Trainer:
 
         B, _, _ = input_ids.shape
 
-        start_logits, end_logits, logits, pred_spans = network(input_ids, attention_mask)
+        hidden_states, attention_mask = network.encode(input_ids, attention_mask)
+        start_logits, end_logits = network.get_token_logits(hidden_states)
+        pred_spans = filter_spans(start_logits, end_logits, attention_mask, network.max_span_len, network.topk_spans, network.keep_neighbor)
+        span_reprs = get_span_reprs(hidden_states, pred_spans)
+        logits = network.get_logits(span_reprs)
         
         gold_list, pred_list = extract_spans(all_spans, all_labels, pred_spans)
         gold_list = list_to_tuple(gold_list)

8_entities_top_span_self_ensemble_no_weight_20/lasts/8_entities_top_span_self_ensemble_no_weight_20_s26092004_f0_last_ema.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2c52d0aa8377ce14c4dc4daf9db1c971b886711672876e1cc2f88bd4d80d18b8
+oid sha256:6d8c6d8cdb0343a8286e705230b1c6c65d137b9bdfb533daecdb805d22dcc40d
 size 554305230

8_entities_top_span_self_ensemble_no_weight_20/r1s/8_entities_top_span_self_ensemble_no_weight_20_s26092004_f0_r1_vs1.69613_ema.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a044d1c538e5ae2c18d2452a9f44819803179f0f5628a82370cb36341392abb3
+size 554320278