File size: 2,744 Bytes
ab331fa | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 | """
data_prep.py - DO NOT RUN THIS SCRIPT!
Run this script, model/model.py and the notebooks through pipeline.py
"""
import json
from pathlib import Path
import numpy as np
import pandas as pd
import pickle
from collections import Counter
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split
from sym_map import SYNONYM_MAP
df = pd.read_csv(r'C:\Portfolio-Projects\Job-Description-Analysis\data\clean\cleaned_job_descriptions.csv')
SKILLS_FIX = {
'tesnorflow/pytorch': 'tensorflow/pytorch',
'numpyhugging face': 'numpy',
'sytem design': 'system design',
'python. ml': 'ml',
}
def normalizer(skills) -> list:
if pd.isna(skills):
return []
skill = [s.strip().lower() for s in skills.split(',') if s.strip()]
fixed = [SKILLS_FIX.get(s, s) for s in skill]
return list(dict.fromkeys(fixed))
df['skill_list'] = df['tech_skills'].apply(normalizer)
freq = Counter(s for lst in df['skill_list'] for s in lst)
VOCAB = sorted([s for s, c in freq.items() if c >= 2])
def encoder(skill_list) -> list:
return [1 if lbl in skill_list else 0 for lbl in VOCAB]
y = np.array([encoder(lst) for lst in df['skill_list']], dtype=np.float32)
def apply_synonyms(text: str) -> str:
text = text.lower()
for phrase, canonical in sorted(SYNONYM_MAP.items(), key=lambda x: -len(x[0])):
text = text.replace(phrase, canonical)
return text
jd_input = (
df['job_desc'].fillna('').apply(apply_synonyms) + ' ' +
df['role'].fillna('').str.lower() + ' ' +
df['type'].fillna('').str.lower()
)
vectorizer = TfidfVectorizer(
max_features=150,
stop_words='english',
ngram_range=(1, 2),
min_df=2,
)
X = vectorizer.fit_transform(jd_input).toarray().astype(np.float32)
X_train, X_test, y_train, y_test, idx_train, idx_test = train_test_split(
X, y, np.arange(len(df)),
test_size=0.2,
random_state=42
)
print(f"Dataset: {len(df)} rows | Vocab: {len(VOCAB)} labels | "
f"TF-IDF features: {X.shape[1]}")
print(f"Train: {len(X_train)} | Test: {len(X_test)}")
# Save artifacts relative to this script location so pipeline.py can be run from repo root
OUT_DIR = Path(__file__).resolve().parent
np.savez(
OUT_DIR / 'prepared_data.npz',
X_train=X_train,
X_test=X_test,
y_train=y_train,
y_test=y_test,
idx_train=idx_train,
idx_test=idx_test,
)
with open(OUT_DIR / 'label_vocab.json', 'w') as f:
json.dump(VOCAB, f, indent=2)
with open(OUT_DIR / 'vectorizer.pkl', 'wb') as f:
pickle.dump(vectorizer, f)
print("Successfully Vectorized and Pickled Data") |