text_preprocessing.js

export const DEFAULT_LIMITS = Object.freeze({ maxChars: 4000, maxBatchItems: 100 });

export function normalizeText(value, maxChars = DEFAULT_LIMITS.maxChars) {
  if (value === null || value === undefined) return '';
  return String(value)
    .normalize('NFC')
    .replace(/\r\n?/g, '\n')
    .replace(/[\t\f\v]+/g, ' ')
    .split('\n')
    .map((line) => line.replace(/ {2,}/g, ' ').trim())
    .join('\n')
    .replace(/\n{3,}/g, '\n\n')
    .trim()
    .slice(0, maxChars);
}

export function splitSentences(text) {
  const cleaned = normalizeText(text);
  if (!cleaned) return [];
  return cleaned
    .split(/(?<=[.!?])\s+(?=[A-Z0-9"'])|\n+/g)
    .map((item) => item.trim())
    .filter(Boolean);
}

export function parseBatchInput(text, maxItems = DEFAULT_LIMITS.maxBatchItems) {
  return normalizeText(text)
    .split(/\n+/)
    .map((item) => item.trim())
    .filter(Boolean)
    .slice(0, maxItems);
}