#!/usr/bin/env bash
set -euo pipefail

if [ "$#" -lt 1 ]; then
  echo "Usage: scripts/prepare_data_and_model.sh DATA_PATH [VOCAB_SIZE] [TOKENS_PER_PARAMETER]"
  exit 2
fi

data_path="$1"
vocab_size="${2:-8192}"
ratio="${3:-20}"

wvy-tokenizer --data "$data_path" --vocab-size "$vocab_size" --output artifacts/tokenizer
wvy-scan --data "$data_path" --tokenizer artifacts/tokenizer --tokens-per-parameter "$ratio" --output artifacts/data_report.json
wvy-build --report artifacts/data_report.json --tokenizer artifacts/tokenizer --output artifacts/model_config.json

echo "Prepared artifacts/tokenizer, artifacts/data_report.json, and artifacts/model_config.json"