Delete wandb

wandb/debug-internal.log

@@ -1,13 +0,0 @@
-{"time":"2026-04-18T12:19:16.854153102Z","level":"INFO","msg":"stream: starting","core version":"0.24.0"}
-{"time":"2026-04-18T12:19:17.275622627Z","level":"INFO","msg":"stream: created new stream","id":"2mk39j3k"}
-{"time":"2026-04-18T12:19:17.275728468Z","level":"INFO","msg":"handler: started","stream_id":"2mk39j3k"}
-{"time":"2026-04-18T12:19:17.27585918Z","level":"INFO","msg":"stream: started","id":"2mk39j3k"}
-{"time":"2026-04-18T12:19:17.275907737Z","level":"INFO","msg":"writer: started","stream_id":"2mk39j3k"}
-{"time":"2026-04-18T12:19:17.275922617Z","level":"INFO","msg":"sender: started","stream_id":"2mk39j3k"}
-{"time":"2026-04-18T12:19:17.416506096Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"}
-{"time":"2026-04-18T16:22:36.413732384Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
-{"time":"2026-04-18T16:22:36.528364306Z","level":"INFO","msg":"handler: operation stats","stats":{}}
-{"time":"2026-04-18T16:22:36.53202978Z","level":"INFO","msg":"stream: closing","id":"2mk39j3k"}
-{"time":"2026-04-18T16:22:36.532057065Z","level":"INFO","msg":"handler: closed","stream_id":"2mk39j3k"}
-{"time":"2026-04-18T16:22:36.532311468Z","level":"INFO","msg":"sender: closed","stream_id":"2mk39j3k"}
-{"time":"2026-04-18T16:22:36.532328723Z","level":"INFO","msg":"stream: closed","id":"2mk39j3k"}

wandb/debug.log

@@ -1,24 +0,0 @@
-2026-04-18 12:19:16,551 INFO    MainThread:4105 [wandb_setup.py:_flush():81] Current SDK version is 0.24.0
-2026-04-18 12:19:16,551 INFO    MainThread:4105 [wandb_setup.py:_flush():81] Configure stats pid to 4105
-2026-04-18 12:19:16,551 INFO    MainThread:4105 [wandb_setup.py:_flush():81] Loading settings from environment variables
-2026-04-18 12:19:16,551 INFO    MainThread:4105 [wandb_init.py:setup_run_log_directory():717] Logging user logs to outputs/2026-04-18/12-19-14/wandb/run-20260418_121916-2mk39j3k/logs/debug.log
-2026-04-18 12:19:16,551 INFO    MainThread:4105 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to outputs/2026-04-18/12-19-14/wandb/run-20260418_121916-2mk39j3k/logs/debug-internal.log
-2026-04-18 12:19:16,551 INFO    MainThread:4105 [wandb_init.py:init():844] calling init triggers
-2026-04-18 12:19:16,551 INFO    MainThread:4105 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
-config: {'model': {'name': 'EleutherAI/pythia-1.4b', 'checkpoint_path': None, 'from_scratch': False}, 'training': {'epochs': 3, 'batch_size': 4, 'eval_batch_size': 12, 'gradient_accumulation_steps': 4, 'lr': 2e-05, 'weight_decay': 0.1, 'betas': [0.9, 0.95], 'eps': 1e-08, 'lr_scheduler': 'wsd', 'warmup_ratio': 0.1, 'decay_ratio': 0.2, 'warmup_steps': 100, 'min_lr_ratio': 0.1, 'max_grad_norm': 1.0, 'use_amp': True, 'resume': False, 'resume_checkpoint': None}, 'data': {'path': '/workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full', 'max_context_len': 4096, 'max_target_len': 256, 'num_workers': 4, 'pin_memory': True}, 'logging': {'log_interval': 10, 'save_interval': 3000, 'eval_interval': 1000, 'save_every_epoch': True}, 'tracking': {'enabled': True, 'backend': 'wandb', 'project': 'code-completion_full', 'run_name': 'pythia_1_4b_v4_lr_2e-5', 'entity': None, 'base_url': 'https://wandb.platun0v.ru', 'local_dir': 'outputs/2026-04-18/12-19-14'}, 'paths': {'output_dir': 'outputs/2026-04-18/12-19-14'}, 'seed': 42, 'device': 'cuda', '_wandb': {'code_path': 'code/code_completion_exp/train_pythia/train.py'}}
-2026-04-18 12:19:16,551 INFO    MainThread:4105 [wandb_init.py:init():892] starting backend
-2026-04-18 12:19:16,824 INFO    MainThread:4105 [wandb_init.py:init():895] sending inform_init request
-2026-04-18 12:19:16,852 INFO    MainThread:4105 [wandb_init.py:init():903] backend started and connected
-2026-04-18 12:19:16,858 INFO    MainThread:4105 [wandb_init.py:init():973] updated telemetry
-2026-04-18 12:19:16,890 INFO    MainThread:4105 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
-2026-04-18 12:19:17,414 INFO    MainThread:4105 [wandb_init.py:init():1044] starting run threads in backend
-2026-04-18 12:19:17,567 INFO    MainThread:4105 [wandb_run.py:_console_start():2529] atexit reg
-2026-04-18 12:19:17,568 INFO    MainThread:4105 [wandb_run.py:_redirect():2377] redirect: wrap_raw
-2026-04-18 12:19:17,568 INFO    MainThread:4105 [wandb_run.py:_redirect():2446] Wrapping output streams.
-2026-04-18 12:19:17,568 INFO    MainThread:4105 [wandb_run.py:_redirect():2469] Redirects installed.
-2026-04-18 12:19:17,571 INFO    MainThread:4105 [wandb_init.py:init():1084] run started, returning control to user process
-2026-04-18 16:22:34,834 INFO    MainThread:4105 [wandb_run.py:_finish():2295] finishing run nikita/code-completion_full/2mk39j3k
-2026-04-18 16:22:34,835 INFO    MainThread:4105 [wandb_run.py:_atexit_cleanup():2494] got exitcode: 0
-2026-04-18 16:22:34,835 INFO    MainThread:4105 [wandb_run.py:_restore():2476] restore
-2026-04-18 16:22:34,835 INFO    MainThread:4105 [wandb_run.py:_restore():2482] restore done
-2026-04-18 16:22:36,531 INFO    MainThread:4105 [wandb_run.py:_footer_sync_info():3870] logging synced files

wandb/run-20260418_121916-2mk39j3k/files/code/code_completion_exp/train_pythia/train.py

@@ -1,598 +0,0 @@
-"""
-Training Pipeline для Pythia (decoder-only transformer) на задаче Code Completion.
-
-Конфигурация через Hydra + OmegaConf, логирование в Trackio.
-Поддержка DDP через Accelerate для multi-GPU тренировки.
-
-Использование:
-    # Базовый запуск (single GPU)
-    python train.py
-
-    # Multi-GPU с Accelerate
-    accelerate launch train.py
-
-    # Multi-GPU с указанием количества GPU
-    accelerate launch --num_processes=4 train.py
-
-    # Переопределение параметров через CLI
-    python train.py training.lr=1e-4 training.epochs=5
-
-    # Выбор другого конфига модели
-    python train.py model=pythia_160m
-
-    # Multirun (sweep)
-    python train.py --multirun training.lr=1e-4,3e-4,1e-3
-
-    # Без логирования
-    python train.py tracking.enabled=false
-"""
-
-import os
-import math
-import time
-from pathlib import Path
-
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from torch.utils.data import DataLoader
-from datasets import load_from_disk
-
-import hydra
-from hydra.core.hydra_config import HydraConfig
-from omegaconf import DictConfig, OmegaConf
-from transformers import (
-    AutoTokenizer,
-    AutoModelForCausalLM,
-    AutoConfig,
-    PreTrainedTokenizerBase,
-)
-from accelerate import Accelerator
-from accelerate.utils import set_seed as accelerate_set_seed
-
-# Ensure repo root is on sys.path (needed when running from subdirectory)
-import sys
-sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
-
-# Shared training library
-from training_lib.utils import AverageMeter, log_message
-from training_lib.checkpointing import save_checkpoint, load_checkpoint
-from training_lib.schedulers import get_lr_scheduler
-from training_lib.tracking import init_tracking, log_metrics, finish_tracking
-from training_lib.validation import run_validation
-
-
-# ============================================================================
-# ДАННЫЕ
-# ============================================================================
-
-
-class CodeCompletionCollator:
-    """Collate function для батчирования примеров code completion."""
-
-    def __init__(
-        self,
-        tokenizer: PreTrainedTokenizerBase,
-        max_context_len: int = 1024,
-        max_target_len: int = 256,
-    ):
-        self.tokenizer = tokenizer
-        self.max_context_len = max_context_len
-        self.max_target_len = max_target_len
-        self.pad_token_id = tokenizer.pad_token_id
-
-    def __call__(self, batch: list[dict]) -> dict:
-        contexts = [item["context"] for item in batch]
-        targets = [item["target"] for item in batch]
-
-        encoded_contexts = self.tokenizer(
-            contexts,
-            add_special_tokens=True,
-            truncation=True,
-            max_length=self.max_context_len,
-            return_tensors=None,
-        )
-        encoded_targets = self.tokenizer(
-            targets,
-            add_special_tokens=False,
-            truncation=True,
-            max_length=self.max_target_len,
-            return_tensors=None,
-        )
-
-        input_ids_list = []
-        context_lengths = []
-
-        for ctx_ids, tgt_ids in zip(
-            encoded_contexts["input_ids"], encoded_targets["input_ids"]
-        ):
-            tgt_ids = tgt_ids + [self.tokenizer.eos_token_id]
-            context_lengths.append(len(ctx_ids))
-            input_ids_list.append(ctx_ids + tgt_ids)
-
-        max_len = max(len(ids) for ids in input_ids_list)
-
-        padded_input_ids = []
-        attention_mask = []
-
-        for ids in input_ids_list:
-            padding_len = max_len - len(ids)
-            padded_input_ids.append(ids + [self.pad_token_id] * padding_len)
-            attention_mask.append([1] * len(ids) + [0] * padding_len)
-
-        return {
-            "input_ids": torch.tensor(padded_input_ids, dtype=torch.long),
-            "attention_mask": torch.tensor(attention_mask, dtype=torch.long),
-            "context_lengths": torch.tensor(context_lengths, dtype=torch.long),
-        }
-
-
-def create_dataloaders(
-    cfg: DictConfig, tokenizer: PreTrainedTokenizerBase
-) -> dict[str, DataLoader]:
-    """Создание DataLoader'ов для train и validation."""
-    dataset_dict = load_from_disk(cfg.data.path)
-
-    collator = CodeCompletionCollator(
-        tokenizer=tokenizer,
-        max_context_len=cfg.data.max_context_len,
-        max_target_len=cfg.data.max_target_len,
-    )
-
-    dataloaders = {}
-
-    if "train" in dataset_dict:
-        dataloaders["train"] = DataLoader(
-            dataset_dict["train"],
-            batch_size=cfg.training.batch_size,
-            shuffle=True,
-            collate_fn=collator,
-            num_workers=cfg.data.num_workers,
-            pin_memory=cfg.data.pin_memory,
-        )
-
-    if "validation" in dataset_dict:
-        eval_batch_size = cfg.training.get("eval_batch_size", cfg.training.batch_size)
-        dataloaders["validation"] = DataLoader(
-            dataset_dict["validation"],
-            batch_size=eval_batch_size,
-            shuffle=False,
-            collate_fn=collator,
-            num_workers=cfg.data.num_workers,
-            pin_memory=cfg.data.pin_memory,
-        )
-
-    return dataloaders
-
-
-
-
-# ============================================================================
-# LOSS ФУНКЦИИ
-# ============================================================================
-
-
-def compute_loss(
-    logits: torch.Tensor,
-    input_ids: torch.Tensor,
-    context_lengths: torch.Tensor,
-    attention_mask: torch.Tensor,
-) -> dict:
-    """Вычисление loss для авторегрессионной модели."""
-    batch_size, seq_len, vocab_size = logits.shape
-
-    shift_logits = logits[:, :-1, :].contiguous()
-    shift_labels = input_ids[:, 1:].contiguous()
-    shift_mask = attention_mask[:, 1:].contiguous()
-
-    target_mask = torch.zeros_like(shift_labels, dtype=torch.bool)
-    for i in range(batch_size):
-        ctx_len = context_lengths[i].item()
-        target_mask[i, ctx_len - 1 :] = True
-
-    final_mask = target_mask & shift_mask.bool()
-
-    if final_mask.sum() > 0:
-        loss = F.cross_entropy(
-            shift_logits[final_mask], shift_labels[final_mask], reduction="mean"
-        )
-    else:
-        loss = torch.tensor(0.0, device=logits.device)
-
-    return {"loss": loss}
-
-
-def _pythia_forward_loss(
-    model: nn.Module,
-    batch: dict,
-    cfg: DictConfig,
-    accelerator: Accelerator,
-) -> dict:
-    """Forward + loss for a plain HF causal LM (attention_mask= kwarg, .logits)."""
-    input_ids = batch["input_ids"]
-    attention_mask = batch["attention_mask"]
-    context_lengths = batch["context_lengths"]
-    output = model(input_ids, attention_mask=attention_mask)
-    return compute_loss(output.logits, input_ids, context_lengths, attention_mask)
-
-
-# ============================================================================
-# PARAMETER GROUPING
-# ============================================================================
-
-
-def group_params(model: nn.Module, weight_decay: float) -> list[dict]:
-    """Группировка параметров для optimizer."""
-    decay_params = []
-    no_decay_params = []
-    
-    for name, param in model.named_parameters():
-        if not param.requires_grad:
-            continue
-            
-        if "bias" in name or "LayerNorm" in name or "layernorm" in name:
-            no_decay_params.append(param)
-        else:
-            decay_params.append(param)
-    
-    return [
-        {"params": decay_params, "weight_decay": weight_decay},
-        {"params": no_decay_params, "weight_decay": 0.0},
-    ]
-
-
-
-
-# ============================================================================
-# TRAINING LOOP
-# ============================================================================
-
-
-def train_epoch(
-    model: nn.Module,
-    dataloader: DataLoader,
-    optimizer: torch.optim.Optimizer,
-    scheduler,
-    cfg: DictConfig,
-    epoch: int,
-    global_step: int,
-    accelerator: Accelerator,
-    val_dataloader: DataLoader | None = None,
-    best_val_loss: float = float("inf"),
-) -> tuple[int, float]:
-    """Один epoch тренировки. Возвращает (global_step, best_val_loss)."""
-    model.train()
-
-    loss_meter = AverageMeter()
-
-    optimizer.zero_grad()
-    accumulated_loss = 0.0
-    accumulated_steps = 0
-
-    epoch_start_time = time.time()
-    step_start_time = time.time()
-
-    for batch_idx, batch in enumerate(dataloader):
-        input_ids = batch["input_ids"]
-        attention_mask = batch["attention_mask"]
-        context_lengths = batch["context_lengths"]
-
-        with accelerator.autocast():
-            output = model(input_ids, attention_mask=attention_mask)
-            logits = output.logits
-            loss_dict = compute_loss(
-                logits, input_ids, context_lengths, attention_mask
-            )
-
-        loss = loss_dict["loss"] / cfg.training.gradient_accumulation_steps
-        accelerator.backward(loss)
-
-        accumulated_loss += loss_dict["loss"].item()
-        accumulated_steps += 1
-
-        if accumulated_steps == cfg.training.gradient_accumulation_steps:
-            if cfg.training.max_grad_norm > 0:
-                accelerator.clip_grad_norm_(
-                    model.parameters(), cfg.training.max_grad_norm
-                )
-
-            optimizer.step()
-            scheduler.step()
-            optimizer.zero_grad()
-
-            avg_loss = accumulated_loss / cfg.training.gradient_accumulation_steps
-            loss_meter.update(avg_loss)
-
-            global_step += 1
-
-            if global_step % cfg.logging.log_interval == 0:
-                step_time = time.time() - step_start_time
-                current_lr = scheduler.get_last_lr()[0]
-
-                metrics = {
-                    "train/loss": loss_meter.val,
-                    "train/loss_avg": loss_meter.avg,
-                    "train/lr": current_lr,
-                    "train/epoch": epoch,
-                    "train/step_time": step_time / cfg.logging.log_interval,
-                }
-
-                log_metrics(metrics, step=global_step)
-
-                log_message(
-                    f"Epoch {epoch} | Step {global_step} | "
-                    f"Loss: {loss_meter.avg:.4f} | "
-                    f"LR: {current_lr:.2e}",
-                    cfg,
-                    accelerator,
-                )
-
-                step_start_time = time.time()
-
-            if (
-                cfg.logging.save_interval > 0
-                and global_step % cfg.logging.save_interval == 0
-            ):
-                save_checkpoint(
-                    model, optimizer, scheduler, global_step, epoch, cfg, accelerator
-                )
-
-            eval_interval = cfg.logging.get("eval_interval", 0)
-            if (
-                eval_interval > 0
-                and val_dataloader is not None
-                and global_step % eval_interval == 0
-            ):
-                val_metrics = run_validation(
-                    model=model,
-                    dataloader=val_dataloader,
-                    cfg=cfg,
-                    global_step=global_step,
-                    accelerator=accelerator,
-                    forward_loss_fn=_pythia_forward_loss,
-                )
-
-                if val_metrics["val/loss"] < best_val_loss:
-                    best_val_loss = val_metrics["val/loss"]
-                    if accelerator.is_main_process:
-                        best_model_path = Path(cfg.paths.output_dir) / "model_best.pt"
-                        unwrapped_model = accelerator.unwrap_model(model)
-                        torch.save(unwrapped_model.state_dict(), best_model_path)
-                        log_message(
-                            f"New best model saved! Val loss: {best_val_loss:.4f}", 
-                            cfg, 
-                            accelerator
-                        )
-
-                        log_metrics(
-                            {
-                                "best/val_loss": best_val_loss,
-                                "best/val_perplexity": val_metrics["val/perplexity"],
-                                "best/step": global_step,
-                            },
-                            step=global_step,
-                        )
-
-                model.train()
-
-            accumulated_loss = 0.0
-            accumulated_steps = 0
-
-    epoch_time = time.time() - epoch_start_time
-    
-    log_message(
-        f"Epoch {epoch} completed in {epoch_time:.2f}s | "
-        f"Loss: {loss_meter.avg:.4f}",
-        cfg,
-        accelerator,
-    )
-
-    log_metrics({
-        "epoch/loss": loss_meter.avg,
-        "epoch/time": epoch_time,
-    })
-
-    return global_step, best_val_loss
-
-
-# ============================================================================
-# MAIN
-# ============================================================================
-
-
-@hydra.main(version_base=None, config_path="configs", config_name="config")
-def main(cfg: DictConfig):
-    """Главная функция тренировки с поддержкой DDP через Accelerate."""
-
-    # === Performance: Enable TF32 for faster matmuls on Ampere+ GPUs ===
-    torch.set_float32_matmul_precision('high')
-
-    # === Accelerator Setup ===
-    mixed_precision = "bf16" if cfg.training.use_amp else "no"
-    
-    accelerator = Accelerator(
-        mixed_precision=mixed_precision,
-        gradient_accumulation_steps=cfg.training.gradient_accumulation_steps,
-    )
-
-    # === Setup ===
-    accelerate_set_seed(cfg.seed)
-    
-    if cfg.paths.output_dir is None:
-        cfg.paths.output_dir = HydraConfig.get().runtime.output_dir
-
-    OmegaConf.resolve(cfg)
-
-    log_message(f"CUDA_VISIBLE_DEVICES: {os.environ.get('CUDA_VISIBLE_DEVICES', 'not set')}", cfg, accelerator)
-    log_message(f"Number of processes: {accelerator.num_processes}", cfg, accelerator)
-    log_message(f"Process index: {accelerator.process_index}", cfg, accelerator)
-    log_message(f"Mixed precision: {mixed_precision}", cfg, accelerator)
-    
-    log_message("=" * 60, cfg, accelerator)
-    log_message("Pythia Training Pipeline (Hydra + Trackio + Accelerate)", cfg, accelerator)
-    log_message("=" * 60, cfg, accelerator)
-    log_message(f"Config:\n{OmegaConf.to_yaml(cfg)}", cfg, accelerator)
-
-    # === Trackio Init ===
-    init_tracking(cfg, accelerator)
-
-    # === Tokenizer ===
-    log_message("Initializing tokenizer...", cfg, accelerator)
-    tokenizer = AutoTokenizer.from_pretrained(cfg.model.name)
-    
-    if tokenizer.pad_token is None:
-        tokenizer.pad_token = tokenizer.eos_token
-        tokenizer.pad_token_id = tokenizer.eos_token_id
-
-    # === Model ===
-    log_message("Loading model...", cfg, accelerator)
-    
-    # Flash Attention 2
-    torch_dtype = torch.bfloat16 if cfg.training.use_amp else torch.float32
-    
-    if cfg.model.checkpoint_path:
-        model = AutoModelForCausalLM.from_pretrained(
-            cfg.model.name,
-            attn_implementation="flash_attention_2",
-            torch_dtype=torch_dtype,
-        )
-        checkpoint = torch.load(cfg.model.checkpoint_path, map_location="cpu")
-        model.load_state_dict(checkpoint["model_state_dict"] if "model_state_dict" in checkpoint else checkpoint)
-        log_message(f"Loaded checkpoint: {cfg.model.checkpoint_path}", cfg, accelerator)
-    elif cfg.model.from_scratch:
-        config = AutoConfig.from_pretrained(cfg.model.name)
-        config._attn_implementation = "flash_attention_2"
-        model = AutoModelForCausalLM.from_config(config, torch_dtype=torch_dtype)
-        log_message(f"Initialized from scratch: {cfg.model.name}", cfg, accelerator)
-    else:
-        model = AutoModelForCausalLM.from_pretrained(
-            cfg.model.name,
-            attn_implementation="flash_attention_2",
-            torch_dtype=torch_dtype,
-        )
-        log_message(f"Loaded pretrained: {cfg.model.name}", cfg, accelerator)
-
-    model.train()
-
-    # Log model info
-    total_params = sum(p.numel() for p in model.parameters())
-    trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
-    log_message(f"Total params: {total_params:,}", cfg, accelerator)
-    log_message(f"Trainable params: {trainable_params:,}", cfg, accelerator)
-
-    # === Data ===
-    log_message("Creating dataloaders...", cfg, accelerator)
-    dataloaders = create_dataloaders(cfg, tokenizer)
-
-    train_dataloader = dataloaders["train"]
-    val_dataloader = dataloaders.get("validation", None)
-
-    log_message(f"Train dataset size: {len(train_dataloader.dataset)}", cfg, accelerator)
-    log_message(f"Train batches per epoch (before DDP split): {len(train_dataloader)}", cfg, accelerator)
-
-    if val_dataloader:
-        log_message(f"Validation dataset size: {len(val_dataloader.dataset)}", cfg, accelerator)
-        log_message(f"Validation batches: {len(val_dataloader)}", cfg, accelerator)
-    else:
-        log_message("No validation dataset found", cfg, accelerator)
-
-    # === Optimizer ===
-    log_message("Creating optimizer...", cfg, accelerator)
-    param_groups = group_params(model, cfg.training.weight_decay)
-
-    optimizer = torch.optim.AdamW(
-        param_groups,
-        lr=cfg.training.lr,
-        betas=tuple(cfg.training.betas),
-        eps=cfg.training.eps,
-    )
-
-    # === Scheduler ===
-    steps_per_epoch = math.ceil(
-        len(train_dataloader) / accelerator.num_processes
-    )
-    total_steps = (
-        cfg.training.epochs
-        * steps_per_epoch
-        // cfg.training.gradient_accumulation_steps
-    )
-    scheduler = get_lr_scheduler(optimizer, cfg, total_steps)
-
-    log_message(
-        f"Total steps: {total_steps}, Steps per epoch: {steps_per_epoch}", 
-        cfg, 
-        accelerator
-    )
-
-    # === Accelerate Prepare ===
-    log_message("Preparing model, optimizer, and dataloaders with Accelerate...", cfg, accelerator)
-    
-    if val_dataloader is not None:
-        model, optimizer, train_dataloader, val_dataloader, scheduler = accelerator.prepare(
-            model, optimizer, train_dataloader, val_dataloader, scheduler
-        )
-    else:
-        model, optimizer, train_dataloader, scheduler = accelerator.prepare(
-            model, optimizer, train_dataloader, scheduler
-        )
-
-    log_message(f"Train batches per epoch (after DDP split): {len(train_dataloader)}", cfg, accelerator)
-
-    # === Resume ===
-    global_step = 0
-    start_epoch = 1
-
-    if cfg.training.resume and cfg.training.resume_checkpoint:
-        global_step, start_epoch = load_checkpoint(
-            model, optimizer, scheduler, cfg.training.resume_checkpoint, cfg, accelerator
-        )
-        start_epoch += 1
-
-    # === Training Loop ===
-    log_message("Starting training...", cfg, accelerator)
-
-    best_val_loss = float("inf")
-
-    try:
-        for epoch in range(start_epoch, cfg.training.epochs + 1):
-            log_message(f"\n{'=' * 60}", cfg, accelerator)
-            log_message(f"EPOCH {epoch}/{cfg.training.epochs}", cfg, accelerator)
-            log_message(f"{'=' * 60}", cfg, accelerator)
-
-            global_step, best_val_loss = train_epoch(
-                model=model,
-                dataloader=train_dataloader,
-                optimizer=optimizer,
-                scheduler=scheduler,
-                cfg=cfg,
-                epoch=epoch,
-                global_step=global_step,
-                accelerator=accelerator,
-                val_dataloader=val_dataloader,
-                best_val_loss=best_val_loss,
-            )
-
-            if cfg.logging.save_every_epoch:
-                save_checkpoint(
-                    model, optimizer, scheduler, global_step, epoch, cfg, accelerator
-                )
-
-    except KeyboardInterrupt:
-        log_message("Training interrupted by user", cfg, accelerator)
-        save_checkpoint(model, optimizer, scheduler, global_step, epoch, cfg, accelerator)
-
-    # === Final Save ===
-    log_message("\nTraining completed!", cfg, accelerator)
-
-    if accelerator.is_main_process:
-        final_model_path = Path(cfg.paths.output_dir) / "model_final.pt"
-        unwrapped_model = accelerator.unwrap_model(model)
-        torch.save(unwrapped_model.state_dict(), final_model_path)
-        log_message(f"Final model: {final_model_path}", cfg, accelerator)
-
-    accelerator.wait_for_everyone()
-    finish_tracking()
-
-
-if __name__ == "__main__":
-    main()

wandb/run-20260418_121916-2mk39j3k/files/config.yaml

@@ -1,126 +0,0 @@
-_wandb:
-    value:
-        cli_version: 0.24.0
-        code_path: code/code_completion_exp/train_pythia/train.py
-        e:
-            lxvl8uvlqbraeb0uteef4wc3ipy2fg2z:
-                codePath: code_completion_exp/train_pythia/train.py
-                codePathLocal: train.py
-                cpu_count: 112
-                cpu_count_logical: 224
-                cudaVersion: "12.9"
-                disk:
-                    /:
-                        total: "244813135872"
-                        used: "43314763776"
-                email: nikita@local.ru
-                executable: /venv/bytellm/bin/python
-                git:
-                    commit: ff609fdb5d8f684fdbf9ea6d64d9440c17614af5
-                    remote: https://github.com/naryst/byte-llms-code.git
-                gpu: NVIDIA H100 80GB HBM3
-                gpu_count: 2
-                gpu_nvidia:
-                    - architecture: Hopper
-                      cudaCores: 16896
-                      memoryTotal: "85520809984"
-                      name: NVIDIA H100 80GB HBM3
-                      uuid: GPU-3c87d2f8-c595-49bd-bb1d-1ebfd19c6fb0
-                    - architecture: Hopper
-                      cudaCores: 16896
-                      memoryTotal: "85520809984"
-                      name: NVIDIA H100 80GB HBM3
-                      uuid: GPU-beb9a6b0-ebef-1f4c-d886-465c96f57ca4
-                host: 3e675e030992
-                memory:
-                    total: "1622968434688"
-                os: Linux-5.15.0-173-generic-x86_64-with-glibc2.39
-                program: /workspace/byte-llms-code/code_completion_exp/train_pythia/train.py
-                python: CPython 3.12.0
-                root: outputs/2026-04-18/12-19-14
-                startedAt: "2026-04-18T12:19:16.549853Z"
-                writerId: lxvl8uvlqbraeb0uteef4wc3ipy2fg2z
-        m: []
-        python_version: 3.12.0
-        t:
-            "1":
-                - 1
-                - 11
-                - 49
-                - 50
-                - 51
-                - 71
-                - 105
-            "2":
-                - 1
-                - 11
-                - 49
-                - 50
-                - 51
-                - 71
-                - 105
-            "3":
-                - 2
-                - 13
-                - 16
-                - 61
-            "4": 3.12.0
-            "5": 0.24.0
-            "6": 4.57.6
-            "12": 0.24.0
-            "13": linux-x86_64
-data:
-    value:
-        max_context_len: 4096
-        max_target_len: 256
-        num_workers: 4
-        path: /workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full
-        pin_memory: true
-device:
-    value: cuda
-logging:
-    value:
-        eval_interval: 1000
-        log_interval: 10
-        save_every_epoch: true
-        save_interval: 3000
-model:
-    value:
-        checkpoint_path: null
-        from_scratch: false
-        name: EleutherAI/pythia-1.4b
-paths:
-    value:
-        output_dir: outputs/2026-04-18/12-19-14
-seed:
-    value: 42
-tracking:
-    value:
-        backend: wandb
-        base_url: https://wandb.platun0v.ru
-        enabled: true
-        entity: null
-        local_dir: outputs/2026-04-18/12-19-14
-        project: code-completion_full
-        run_name: pythia_1_4b_v4_lr_2e-5
-training:
-    value:
-        batch_size: 4
-        betas:
-            - 0.9
-            - 0.95
-        decay_ratio: 0.2
-        epochs: 3
-        eps: 1e-08
-        eval_batch_size: 12
-        gradient_accumulation_steps: 4
-        lr: 2e-05
-        lr_scheduler: wsd
-        max_grad_norm: 1
-        min_lr_ratio: 0.1
-        resume: false
-        resume_checkpoint: null
-        use_amp: true
-        warmup_ratio: 0.1
-        warmup_steps: 100
-        weight_decay: 0.1

wandb/run-20260418_121916-2mk39j3k/files/requirements.txt

@@ -1,246 +0,0 @@
-setuptools==78.1.1
-wheel==0.45.1
-pip==25.2
-webencodings==0.5.1
-triton==3.2.0
-pytz==2025.2
-pydub==0.25.1
-pure_eval==0.2.3
-ptyprocess==0.7.0
-nvidia-ml-py==13.590.48
-nvidia-cusparselt-cu12==0.6.2
-mpmath==1.3.0
-ipython-genutils==0.2.0
-fastjsonschema==2.21.2
-brotli==1.2.0
-antlr4-python3-runtime==4.9.3
-xxhash==3.6.0
-widgetsnbextension==4.0.14
-websocket-client==1.9.0
-webcolors==24.11.1
-wcwidth==0.2.14
-urllib3==2.5.0
-uri-template==1.3.0
-tzdata==2025.2
-typing_extensions==4.15.0
-types-python-dateutil==2.9.0.20251008
-traitlets==5.14.3
-tqdm==4.67.1
-tornado==6.5.2
-tomlkit==0.13.3
-tinycss2==1.4.0
-tabulate==0.9.0
-sympy==1.13.1
-soupsieve==2.8
-sniffio==1.3.1
-smmap==5.0.2
-six==1.17.0
-shellingham==1.5.4
-Send2Trash==1.8.3
-semantic-version==2.10.0
-safetensors==0.6.2
-rpds-py==0.27.1
-rfc3986-validator==0.1.1
-regex==2025.9.18
-pyzmq==27.1.0
-PyYAML==6.0.3
-python-multipart==0.0.22
-python-json-logger==4.0.0
-python-dotenv==1.2.1
-pyparsing==3.2.5
-PyJWT==2.8.0
-Pygments==2.19.2
-pycparser==2.23
-pyarrow==22.0.0
-psutil==7.1.0
-protobuf==6.33.4
-propcache==0.4.1
-prometheus_client==0.23.1
-portalocker==3.2.0
-platformdirs==4.5.0
-pillow==11.3.0
-pexpect==4.9.0
-pathspec==1.0.4
-parso==0.8.5
-pandocfilters==1.5.1
-packaging==25.0
-orjson==3.11.6
-opt_einsum==3.4.0
-nvidia-nvtx-cu12==12.4.127
-nvidia-nvjitlink-cu12==12.4.127
-nvidia-nccl-cu12==2.21.5
-nvidia-curand-cu12==10.3.5.147
-nvidia-cufile-cu12==1.13.1.3
-nvidia-cufft-cu12==11.2.1.3
-nvidia-cuda-runtime-cu12==12.4.127
-nvidia-cuda-nvrtc-cu12==12.4.127
-nvidia-cuda-cupti-cu12==12.4.127
-nvidia-cublas-cu12==12.4.5.8
-numpy==2.3.3
-ninja==1.13.0
-networkx==3.5
-nest-asyncio==1.6.0
-narwhals==2.15.0
-mypy_extensions==1.1.0
-multidict==6.7.0
-mistune==3.1.4
-mdurl==0.1.2
-MarkupSafe==3.0.3
-lxml==6.0.2
-librt==0.8.0
-lark==1.3.0
-kiwisolver==1.4.9
-jupyterlab_widgets==3.0.15
-jupyterlab_pygments==0.3.0
-jsonpointer==3.0.0
-json5==0.12.1
-itsdangerous==2.2.0
-idna==3.10
-hf-xet==1.1.10
-h11==0.16.0
-groovy==0.1.2
-fsspec==2025.9.0
-frozenlist==1.8.0
-fqdn==1.5.1
-fonttools==4.60.1
-filelock==3.19.1
-ffmpy==1.0.0
-executing==2.2.1
-einops==0.8.1
-dill==0.4.0
-defusedxml==0.7.1
-decorator==5.2.1
-debugpy==1.8.17
-dacite==1.9.2
-cycler==0.12.1
-comm==0.2.3
-colorama==0.4.6
-click==8.3.1
-charset-normalizer==3.4.3
-certifi==2025.10.5
-bleach==6.2.0
-babel==2.17.0
-attrs==25.4.0
-async-lru==2.0.5
-asttokens==3.0.0
-annotated-types==0.7.0
-annotated-doc==0.0.4
-aiohappyeyeballs==2.6.1
-aiofiles==24.1.0
-yarl==1.22.0
-uvicorn==0.40.0
-typing-inspection==0.4.2
-terminado==0.18.1
-stack-data==0.6.3
-sentry-sdk==2.50.0
-scipy==1.17.0
-sacrebleu==2.6.0
-rfc3987-syntax==1.1.0
-rfc3339-validator==0.1.4
-requests==2.32.5
-reportlab==4.4.9
-referencing==0.36.2
-python-dateutil==2.9.0.post0
-pydantic_core==2.41.5
-prompt_toolkit==3.0.52
-plotly==6.5.2
-pathlib2==2.3.7.post1
-orderedmultidict==1.0.2
-optree==0.17.0
-omegaconf==2.3.0
-nvidia-cusparse-cu12==12.3.1.170
-nvidia-cudnn-cu12==9.1.0.70
-mypy==1.19.1
-multiprocess==0.70.16
-matplotlib-inline==0.1.7
-markdown-it-py==4.0.0
-jupyter_core==5.8.1
-Jinja2==3.1.6
-jedi==0.19.2
-ipython_pygments_lexers==1.1.1
-httpcore==1.0.9
-gitdb==4.0.12
-ftfy==6.3.1
-contourpy==1.3.3
-cffi==2.0.0
-beautifulsoup4==4.14.2
-anyio==4.11.0
-aiosignal==1.4.0
-starlette==0.50.0
-rich==14.2.0
-pydantic==2.12.5
-pandas==2.3.3
-nvidia-cusolver-cu12==11.6.1.9
-matplotlib==3.10.7
-jupyter_server_terminals==0.5.3
-jupyter_client==8.6.3
-jsonschema-specifications==2025.9.1
-ipython==9.6.0
-hydra-core==1.3.2
-huggingface-hub==0.35.3
-httpx==0.28.1
-GitPython==3.1.46
-furl==2.1.4
-cryptography==46.0.4
-arrow==1.3.0
-argon2-cffi-bindings==25.1.0
-aiohttp==3.13.1
-wandb==0.24.0
-typer==0.21.1
-torch==2.6.0
-tokenizers==0.22.1
-seaborn==0.13.2
-safehttpx==0.1.7
-jsonschema==4.25.1
-joypy==0.2.6
-isoduration==20.11.0
-ipywidgets==8.1.7
-ipykernel==6.30.1
-gradio_client==2.0.3
-fastapi==0.128.0
-Authlib==1.6.6
-argon2-cffi==25.1.0
-transformers==4.57.6
-nbformat==5.10.4
-mlstm_kernels==2.0.2
-jupyter-console==6.6.3
-gradio==6.5.1
-datasets==4.3.0
-clearml==1.16.4
-accelerate==1.10.1
-xlstm==2.0.4
-nbclient==0.10.2
-jupyter-events==0.12.0
-trackio==0.15.0
-nbconvert==7.16.6
-jupyter_server==2.17.0
-notebook_shim==0.2.4
-jupyterlab_server==2.27.3
-jupyter-lsp==2.3.0
-nbclassic==1.3.3
-jupyterlab==4.4.9
-notebook==7.4.7
-jupyter_contrib_core==0.4.2
-jupyter==1.1.1
-jupyter_nbextensions_configurator==0.6.4
-causal-conv1d==1.5.0.post8
-flash_attn==2.7.4.post1
-mamba-ssm==2.2.4
-hnet==0.0.1
-speedtest-cli==2.1.3
-autocommand==2.2.2
-backports.tarfile==1.2.0
-importlib_metadata==8.0.0
-inflect==7.3.1
-jaraco.collections==5.1.0
-jaraco.context==5.3.0
-jaraco.functools==4.0.1
-jaraco.text==3.12.1
-more-itertools==10.3.0
-packaging==24.2
-platformdirs==4.2.2
-tomli==2.0.1
-typeguard==4.3.0
-typing_extensions==4.12.2
-wheel==0.45.1
-zipp==3.19.2

wandb/run-20260418_121916-2mk39j3k/files/wandb-metadata.json

@@ -1,47 +0,0 @@
-{
-  "os": "Linux-5.15.0-173-generic-x86_64-with-glibc2.39",
-  "python": "CPython 3.12.0",
-  "startedAt": "2026-04-18T12:19:16.549853Z",
-  "program": "/workspace/byte-llms-code/code_completion_exp/train_pythia/train.py",
-  "codePath": "code_completion_exp/train_pythia/train.py",
-  "codePathLocal": "train.py",
-  "git": {
-    "remote": "https://github.com/naryst/byte-llms-code.git",
-    "commit": "ff609fdb5d8f684fdbf9ea6d64d9440c17614af5"
-  },
-  "email": "nikita@local.ru",
-  "root": "outputs/2026-04-18/12-19-14",
-  "host": "3e675e030992",
-  "executable": "/venv/bytellm/bin/python",
-  "cpu_count": 112,
-  "cpu_count_logical": 224,
-  "gpu": "NVIDIA H100 80GB HBM3",
-  "gpu_count": 2,
-  "disk": {
-    "/": {
-      "total": "244813135872",
-      "used": "43314763776"
-    }
-  },
-  "memory": {
-    "total": "1622968434688"
-  },
-  "gpu_nvidia": [
-    {
-      "name": "NVIDIA H100 80GB HBM3",
-      "memoryTotal": "85520809984",
-      "cudaCores": 16896,
-      "architecture": "Hopper",
-      "uuid": "GPU-3c87d2f8-c595-49bd-bb1d-1ebfd19c6fb0"
-    },
-    {
-      "name": "NVIDIA H100 80GB HBM3",
-      "memoryTotal": "85520809984",
-      "cudaCores": 16896,
-      "architecture": "Hopper",
-      "uuid": "GPU-beb9a6b0-ebef-1f4c-d886-465c96f57ca4"
-    }
-  ],
-  "cudaVersion": "12.9",
-  "writerId": "lxvl8uvlqbraeb0uteef4wc3ipy2fg2z"
-}

wandb/run-20260418_121916-2mk39j3k/files/wandb-summary.json

@@ -1 +0,0 @@
-{"train/lr":2.0000000000000003e-06,"train/loss":0.6684185862541199,"val/time":125.04461622238159,"_runtime":14595,"_timestamp":1.7765293278895276e+09,"_step":29660,"epoch/loss":0.7010961592882488,"val/loss":1.0707972391764677,"_wandb":{"runtime":14595},"epoch/time":4867.094700336456,"best/val_perplexity":2.88394085342274,"best/step":19000,"train/loss_avg":0.7010793855010286,"train/step_time":0.357418155670166,"val/perplexity":2.9409693343660885,"best/val_loss":1.0516192755105014,"train/epoch":3}

wandb/run-20260418_121916-2mk39j3k/logs/debug-core.log

@@ -1,16 +0,0 @@
-{"time":"2026-04-18T12:19:16.63631091Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp8o4e0c4r/port-4105.txt","pid":4105,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
-{"time":"2026-04-18T12:19:16.636897052Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":4105}
-{"time":"2026-04-18T12:19:16.636890378Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-4105-4125-1730369648/socket","Net":"unix"}}
-{"time":"2026-04-18T12:19:16.824047472Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
-{"time":"2026-04-18T12:19:16.853988313Z","level":"INFO","msg":"handleInformInit: received","streamId":"2mk39j3k","id":"1(@)"}
-{"time":"2026-04-18T12:19:17.275867326Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"2mk39j3k","id":"1(@)"}
-{"time":"2026-04-18T16:22:36.531964538Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"2mk39j3k","id":"1(@)"}
-{"time":"2026-04-18T16:22:36.532983895Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"2mk39j3k","id":"1(@)"}
-{"time":"2026-04-18T16:22:36.553203529Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
-{"time":"2026-04-18T16:22:36.553260163Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
-{"time":"2026-04-18T16:22:36.553277979Z","level":"INFO","msg":"server is shutting down"}
-{"time":"2026-04-18T16:22:36.553320826Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
-{"time":"2026-04-18T16:22:36.553421428Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
-{"time":"2026-04-18T16:22:36.553434515Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
-{"time":"2026-04-18T16:22:36.553408703Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-4105-4125-1730369648/socket","Net":"unix"}}
-{"time":"2026-04-18T16:22:36.553462952Z","level":"INFO","msg":"server is closed"}

wandb/run-20260418_121916-2mk39j3k/logs/debug-internal.log

@@ -1,13 +0,0 @@
-{"time":"2026-04-18T12:19:16.854153102Z","level":"INFO","msg":"stream: starting","core version":"0.24.0"}
-{"time":"2026-04-18T12:19:17.275622627Z","level":"INFO","msg":"stream: created new stream","id":"2mk39j3k"}
-{"time":"2026-04-18T12:19:17.275728468Z","level":"INFO","msg":"handler: started","stream_id":"2mk39j3k"}
-{"time":"2026-04-18T12:19:17.27585918Z","level":"INFO","msg":"stream: started","id":"2mk39j3k"}
-{"time":"2026-04-18T12:19:17.275907737Z","level":"INFO","msg":"writer: started","stream_id":"2mk39j3k"}
-{"time":"2026-04-18T12:19:17.275922617Z","level":"INFO","msg":"sender: started","stream_id":"2mk39j3k"}
-{"time":"2026-04-18T12:19:17.416506096Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"}
-{"time":"2026-04-18T16:22:36.413732384Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
-{"time":"2026-04-18T16:22:36.528364306Z","level":"INFO","msg":"handler: operation stats","stats":{}}
-{"time":"2026-04-18T16:22:36.53202978Z","level":"INFO","msg":"stream: closing","id":"2mk39j3k"}
-{"time":"2026-04-18T16:22:36.532057065Z","level":"INFO","msg":"handler: closed","stream_id":"2mk39j3k"}
-{"time":"2026-04-18T16:22:36.532311468Z","level":"INFO","msg":"sender: closed","stream_id":"2mk39j3k"}
-{"time":"2026-04-18T16:22:36.532328723Z","level":"INFO","msg":"stream: closed","id":"2mk39j3k"}

wandb/run-20260418_121916-2mk39j3k/logs/debug.log

@@ -1,24 +0,0 @@
-2026-04-18 12:19:16,551 INFO    MainThread:4105 [wandb_setup.py:_flush():81] Current SDK version is 0.24.0
-2026-04-18 12:19:16,551 INFO    MainThread:4105 [wandb_setup.py:_flush():81] Configure stats pid to 4105
-2026-04-18 12:19:16,551 INFO    MainThread:4105 [wandb_setup.py:_flush():81] Loading settings from environment variables
-2026-04-18 12:19:16,551 INFO    MainThread:4105 [wandb_init.py:setup_run_log_directory():717] Logging user logs to outputs/2026-04-18/12-19-14/wandb/run-20260418_121916-2mk39j3k/logs/debug.log
-2026-04-18 12:19:16,551 INFO    MainThread:4105 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to outputs/2026-04-18/12-19-14/wandb/run-20260418_121916-2mk39j3k/logs/debug-internal.log
-2026-04-18 12:19:16,551 INFO    MainThread:4105 [wandb_init.py:init():844] calling init triggers
-2026-04-18 12:19:16,551 INFO    MainThread:4105 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
-config: {'model': {'name': 'EleutherAI/pythia-1.4b', 'checkpoint_path': None, 'from_scratch': False}, 'training': {'epochs': 3, 'batch_size': 4, 'eval_batch_size': 12, 'gradient_accumulation_steps': 4, 'lr': 2e-05, 'weight_decay': 0.1, 'betas': [0.9, 0.95], 'eps': 1e-08, 'lr_scheduler': 'wsd', 'warmup_ratio': 0.1, 'decay_ratio': 0.2, 'warmup_steps': 100, 'min_lr_ratio': 0.1, 'max_grad_norm': 1.0, 'use_amp': True, 'resume': False, 'resume_checkpoint': None}, 'data': {'path': '/workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full', 'max_context_len': 4096, 'max_target_len': 256, 'num_workers': 4, 'pin_memory': True}, 'logging': {'log_interval': 10, 'save_interval': 3000, 'eval_interval': 1000, 'save_every_epoch': True}, 'tracking': {'enabled': True, 'backend': 'wandb', 'project': 'code-completion_full', 'run_name': 'pythia_1_4b_v4_lr_2e-5', 'entity': None, 'base_url': 'https://wandb.platun0v.ru', 'local_dir': 'outputs/2026-04-18/12-19-14'}, 'paths': {'output_dir': 'outputs/2026-04-18/12-19-14'}, 'seed': 42, 'device': 'cuda', '_wandb': {'code_path': 'code/code_completion_exp/train_pythia/train.py'}}
-2026-04-18 12:19:16,551 INFO    MainThread:4105 [wandb_init.py:init():892] starting backend
-2026-04-18 12:19:16,824 INFO    MainThread:4105 [wandb_init.py:init():895] sending inform_init request
-2026-04-18 12:19:16,852 INFO    MainThread:4105 [wandb_init.py:init():903] backend started and connected
-2026-04-18 12:19:16,858 INFO    MainThread:4105 [wandb_init.py:init():973] updated telemetry
-2026-04-18 12:19:16,890 INFO    MainThread:4105 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
-2026-04-18 12:19:17,414 INFO    MainThread:4105 [wandb_init.py:init():1044] starting run threads in backend
-2026-04-18 12:19:17,567 INFO    MainThread:4105 [wandb_run.py:_console_start():2529] atexit reg
-2026-04-18 12:19:17,568 INFO    MainThread:4105 [wandb_run.py:_redirect():2377] redirect: wrap_raw
-2026-04-18 12:19:17,568 INFO    MainThread:4105 [wandb_run.py:_redirect():2446] Wrapping output streams.
-2026-04-18 12:19:17,568 INFO    MainThread:4105 [wandb_run.py:_redirect():2469] Redirects installed.
-2026-04-18 12:19:17,571 INFO    MainThread:4105 [wandb_init.py:init():1084] run started, returning control to user process
-2026-04-18 16:22:34,834 INFO    MainThread:4105 [wandb_run.py:_finish():2295] finishing run nikita/code-completion_full/2mk39j3k
-2026-04-18 16:22:34,835 INFO    MainThread:4105 [wandb_run.py:_atexit_cleanup():2494] got exitcode: 0
-2026-04-18 16:22:34,835 INFO    MainThread:4105 [wandb_run.py:_restore():2476] restore
-2026-04-18 16:22:34,835 INFO    MainThread:4105 [wandb_run.py:_restore():2482] restore done
-2026-04-18 16:22:36,531 INFO    MainThread:4105 [wandb_run.py:_footer_sync_info():3870] logging synced files

wandb/run-20260418_121916-2mk39j3k/run-2mk39j3k.wandb

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c9aa82c70ac11e1e1ec7ed8e2f4354f65bceddca985f5a275dd53cffb1973aae
-size 4312629