Spaces:
Sleeping
Sleeping
File size: 1,338 Bytes
72e2b6e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 | from pathlib import Path
import pandas as pd
from datasets import load_dataset
def load_clinc150(subset: str = "plus") -> dict[str, pd.DataFrame]:
dataset = load_dataset("clinc/clinc_oos", subset)
intent_feature = dataset["train"].features["intent"]
splits = {}
for split_name in ["train", "validation", "test"]:
records = [
{
"text": item["text"],
"label": intent_feature.int2str(item["intent"]),
}
for item in dataset[split_name]
]
splits[split_name] = pd.DataFrame(records)
return splits
def save_splits(splits: dict[str, pd.DataFrame], save_dir: str = "data/raw") -> None:
path = Path(save_dir)
path.mkdir(parents=True, exist_ok=True)
for split_name, df in splits.items():
df.to_csv(path / f"{split_name}.csv", index=False)
def load_splits(load_dir: str = "data/raw") -> dict[str, pd.DataFrame]:
path = Path(load_dir)
split_names = ["train", "validation", "test"]
splits = {}
for split_name in split_names:
file_path = path / f"{split_name}.csv"
if not file_path.exists():
raise FileNotFoundError(f"Split not found: {file_path}")
splits[split_name] = pd.read_csv(file_path)
return splits
|