Spaces:
Running
Running
| ; | |
| /* | |
| ============================================================ | |
| CONFIG | |
| ============================================================ | |
| */ | |
| const HF_API = | |
| "https://huggingface.co/api"; | |
| let selectedDatasets = new Set(); | |
| let localFiles = []; | |
| let generatedTrainBlob = null; | |
| let generatedTrainUrl = null; | |
| let generatedRepoUrl = null; | |
| /* | |
| ============================================================ | |
| DOM | |
| ============================================================ | |
| */ | |
| const $ = (id) => | |
| document.getElementById(id); | |
| const searchInput = | |
| $("searchInput"); | |
| const searchButton = | |
| $("searchButton"); | |
| const searchStatus = | |
| $("searchStatus"); | |
| const datasetResults = | |
| $("datasetResults"); | |
| const selectedDatasetsBox = | |
| $("selectedDatasets"); | |
| const manualDatasets = | |
| $("manualDatasets"); | |
| const fileInput = | |
| $("fileInput"); | |
| const fileList = | |
| $("fileList"); | |
| const hfToken = | |
| $("hfToken"); | |
| const buildButton = | |
| $("buildButton"); | |
| const progressSection = | |
| $("progressSection"); | |
| const progressBar = | |
| $("progressBar"); | |
| const progressPercent = | |
| $("progressPercent"); | |
| const progressText = | |
| $("progressText"); | |
| const logBox = | |
| $("log"); | |
| const resultBox = | |
| $("result"); | |
| const resultInfo = | |
| $("resultInfo"); | |
| const downloadTrain = | |
| $("downloadTrain"); | |
| const repoButton = | |
| $("repoButton"); | |
| /* | |
| ============================================================ | |
| LOGGING | |
| ============================================================ | |
| */ | |
| function log(message) { | |
| const time = | |
| new Date().toLocaleTimeString(); | |
| logBox.textContent += | |
| `[${time}] ${message}\n`; | |
| logBox.scrollTop = | |
| logBox.scrollHeight; | |
| } | |
| function clearLog() { | |
| logBox.textContent = ""; | |
| } | |
| /* | |
| ============================================================ | |
| PROGRESS | |
| ============================================================ | |
| */ | |
| function setProgress( | |
| percent, | |
| text | |
| ) { | |
| percent = | |
| Math.max( | |
| 0, | |
| Math.min( | |
| 100, | |
| percent | |
| ) | |
| ); | |
| progressBar.style.width = | |
| `${percent}%`; | |
| progressPercent.textContent = | |
| `${Math.round(percent)}%`; | |
| progressText.textContent = | |
| text || ""; | |
| } | |
| /* | |
| ============================================================ | |
| HF SEARCH | |
| ============================================================ | |
| */ | |
| searchButton.addEventListener( | |
| "click", | |
| searchDatasets | |
| ); | |
| searchInput.addEventListener( | |
| "keydown", | |
| event => { | |
| if ( | |
| event.key === "Enter" | |
| ) { | |
| searchDatasets(); | |
| } | |
| } | |
| ); | |
| async function searchDatasets() { | |
| const query = | |
| searchInput.value.trim(); | |
| if (!query) { | |
| searchStatus.textContent = | |
| "عبارت جستجو را وارد کنید."; | |
| return; | |
| } | |
| searchButton.disabled = true; | |
| datasetResults.innerHTML = | |
| ""; | |
| searchStatus.textContent = | |
| "در حال جستجو..."; | |
| try { | |
| const token = | |
| hfToken.value.trim(); | |
| const headers = {}; | |
| if (token) { | |
| headers.Authorization = | |
| `Bearer ${token}`; | |
| } | |
| const url = | |
| `${HF_API}/datasets?search=${encodeURIComponent(query)}&limit=30&sort=downloads&direction=-1`; | |
| const response = | |
| await fetch( | |
| url, | |
| { | |
| headers | |
| } | |
| ); | |
| if (!response.ok) { | |
| throw new Error( | |
| `HTTP ${response.status}` | |
| ); | |
| } | |
| const datasets = | |
| await response.json(); | |
| renderDatasetResults( | |
| datasets | |
| ); | |
| searchStatus.textContent = | |
| `${datasets.length} دیتاست پیدا شد.`; | |
| } catch (error) { | |
| searchStatus.textContent = | |
| `خطا: ${error.message}`; | |
| } finally { | |
| searchButton.disabled = | |
| false; | |
| } | |
| } | |
| /* | |
| ============================================================ | |
| RENDER SEARCH RESULTS | |
| ============================================================ | |
| */ | |
| function renderDatasetResults( | |
| datasets | |
| ) { | |
| datasetResults.innerHTML = | |
| ""; | |
| for ( | |
| const dataset | |
| of datasets | |
| ) { | |
| const id = | |
| dataset.id; | |
| if (!id) { | |
| continue; | |
| } | |
| const downloads = | |
| Number( | |
| dataset.downloads || 0 | |
| ); | |
| const likes = | |
| Number( | |
| dataset.likes || 0 | |
| ); | |
| const wrapper = | |
| document.createElement( | |
| "label" | |
| ); | |
| wrapper.className = | |
| "dataset-item"; | |
| const checkbox = | |
| document.createElement( | |
| "input" | |
| ); | |
| checkbox.type = | |
| "checkbox"; | |
| checkbox.checked = | |
| selectedDatasets.has( | |
| id | |
| ); | |
| checkbox.addEventListener( | |
| "change", | |
| () => { | |
| if ( | |
| checkbox.checked | |
| ) { | |
| selectedDatasets.add( | |
| id | |
| ); | |
| } else { | |
| selectedDatasets.delete( | |
| id | |
| ); | |
| } | |
| renderSelectedDatasets(); | |
| } | |
| ); | |
| const info = | |
| document.createElement( | |
| "div" | |
| ); | |
| info.className = | |
| "dataset-info"; | |
| const name = | |
| document.createElement( | |
| "div" | |
| ); | |
| name.className = | |
| "dataset-name"; | |
| name.textContent = | |
| id; | |
| const meta = | |
| document.createElement( | |
| "div" | |
| ); | |
| meta.className = | |
| "dataset-meta"; | |
| meta.textContent = | |
| `${downloads.toLocaleString()} downloads · ${likes.toLocaleString()} likes`; | |
| info.appendChild( | |
| name | |
| ); | |
| info.appendChild( | |
| meta | |
| ); | |
| wrapper.appendChild( | |
| checkbox | |
| ); | |
| wrapper.appendChild( | |
| info | |
| ); | |
| datasetResults.appendChild( | |
| wrapper | |
| ); | |
| } | |
| } | |
| /* | |
| ============================================================ | |
| SELECTED DATASETS | |
| ============================================================ | |
| */ | |
| function renderSelectedDatasets() { | |
| selectedDatasetsBox.innerHTML = | |
| ""; | |
| if ( | |
| selectedDatasets.size === 0 | |
| ) { | |
| const empty = | |
| document.createElement( | |
| "span" | |
| ); | |
| empty.className = | |
| "muted"; | |
| empty.textContent = | |
| "هنوز دیتاستی انتخاب نشده"; | |
| selectedDatasetsBox.appendChild( | |
| empty | |
| ); | |
| return; | |
| } | |
| for ( | |
| const dataset | |
| of selectedDatasets | |
| ) { | |
| const chip = | |
| document.createElement( | |
| "span" | |
| ); | |
| chip.className = | |
| "chip"; | |
| chip.textContent = | |
| dataset; | |
| selectedDatasetsBox.appendChild( | |
| chip | |
| ); | |
| } | |
| } | |
| /* | |
| ============================================================ | |
| LOCAL FILES | |
| ============================================================ | |
| */ | |
| fileInput.addEventListener( | |
| "change", | |
| () => { | |
| localFiles = | |
| Array.from( | |
| fileInput.files || [] | |
| ); | |
| renderFiles(); | |
| } | |
| ); | |
| function renderFiles() { | |
| fileList.innerHTML = | |
| ""; | |
| for ( | |
| const file | |
| of localFiles | |
| ) { | |
| const item = | |
| document.createElement( | |
| "div" | |
| ); | |
| item.className = | |
| "file-item"; | |
| const size = | |
| formatBytes( | |
| file.size | |
| ); | |
| item.textContent = | |
| `${file.name} · ${size}`; | |
| fileList.appendChild( | |
| item | |
| ); | |
| } | |
| } | |
| function formatBytes( | |
| bytes | |
| ) { | |
| if ( | |
| bytes < 1024 | |
| ) { | |
| return `${bytes} B`; | |
| } | |
| if ( | |
| bytes < 1024 * 1024 | |
| ) { | |
| return ( | |
| `${( | |
| bytes / | |
| 1024 | |
| ).toFixed(1)} KB` | |
| ); | |
| } | |
| if ( | |
| bytes < 1024 * 1024 * 1024 | |
| ) { | |
| return ( | |
| `${( | |
| bytes / | |
| 1024 / | |
| 1024 | |
| ).toFixed(1)} MB` | |
| ); | |
| } | |
| return ( | |
| `${( | |
| bytes / | |
| 1024 / | |
| 1024 / | |
| 1024 | |
| ).toFixed(2)} GB` | |
| ); | |
| } | |
| /* | |
| ============================================================ | |
| JSON / JSONL PARSER | |
| ============================================================ | |
| */ | |
| async function parseFile( | |
| file | |
| ) { | |
| const text = | |
| await file.text(); | |
| const clean = | |
| text | |
| .replace( | |
| /^\uFEFF/, | |
| "" | |
| ) | |
| .trim(); | |
| if (!clean) { | |
| return []; | |
| } | |
| /* | |
| Try normal JSON first. | |
| */ | |
| try { | |
| const obj = | |
| JSON.parse( | |
| clean | |
| ); | |
| if ( | |
| Array.isArray(obj) | |
| ) { | |
| return obj; | |
| } | |
| if ( | |
| obj && | |
| Array.isArray( | |
| obj.data | |
| ) | |
| ) { | |
| return obj.data; | |
| } | |
| if ( | |
| obj && | |
| typeof obj === | |
| "object" | |
| ) { | |
| return [obj]; | |
| } | |
| } catch (_) { | |
| // Continue as JSONL. | |
| } | |
| /* | |
| JSONL | |
| */ | |
| const records = []; | |
| const lines = | |
| clean.split( | |
| /\r?\n/ | |
| ); | |
| for ( | |
| let i = 0; | |
| i < lines.length; | |
| i++ | |
| ) { | |
| const line = | |
| lines[i].trim(); | |
| if (!line) { | |
| continue; | |
| } | |
| try { | |
| const obj = | |
| JSON.parse( | |
| line | |
| ); | |
| if ( | |
| Array.isArray(obj) | |
| ) { | |
| for ( | |
| const item | |
| of obj | |
| ) { | |
| if ( | |
| item && | |
| typeof item === | |
| "object" | |
| ) { | |
| records.push( | |
| item | |
| ); | |
| } | |
| } | |
| } else { | |
| records.push( | |
| obj | |
| ); | |
| } | |
| } catch (error) { | |
| throw new Error( | |
| `${file.name}: خطا در خط ${i + 1}` | |
| ); | |
| } | |
| } | |
| return records; | |
| } | |
| /* | |
| ============================================================ | |
| NORMALIZATION | |
| ============================================================ | |
| */ | |
| function normalizeText( | |
| value | |
| ) { | |
| if ( | |
| value === null || | |
| value === undefined | |
| ) { | |
| return ""; | |
| } | |
| return String( | |
| value | |
| ) | |
| .replace( | |
| /ي/g, | |
| "ی" | |
| ) | |
| .replace( | |
| /ى/g, | |
| "ی" | |
| ) | |
| .replace( | |
| /ك/g, | |
| "ک" | |
| ) | |
| .replace( | |
| /\s+/g, | |
| " " | |
| ) | |
| .trim(); | |
| } | |
| function stringifyValue( | |
| value | |
| ) { | |
| if ( | |
| typeof value === | |
| "string" | |
| ) { | |
| return value; | |
| } | |
| try { | |
| return JSON.stringify( | |
| value | |
| ); | |
| } catch (_) { | |
| return String( | |
| value | |
| ); | |
| } | |
| } | |
| function normalizeRecord( | |
| record | |
| ) { | |
| if ( | |
| !record || | |
| typeof record !== | |
| "object" | |
| ) { | |
| return null; | |
| } | |
| /* | |
| messages | |
| */ | |
| if ( | |
| Array.isArray( | |
| record.messages | |
| ) | |
| ) { | |
| const messages = []; | |
| for ( | |
| const message | |
| of record.messages | |
| ) { | |
| if ( | |
| !message || | |
| typeof message !== | |
| "object" | |
| ) { | |
| continue; | |
| } | |
| const role = | |
| message.role; | |
| const content = | |
| normalizeText( | |
| stringifyValue( | |
| message.content | |
| ) | |
| ); | |
| if ( | |
| role && | |
| content | |
| ) { | |
| messages.push( | |
| { | |
| role: | |
| String( | |
| role | |
| ), | |
| content | |
| } | |
| ); | |
| } | |
| } | |
| if ( | |
| messages.length | |
| ) { | |
| return { | |
| messages | |
| }; | |
| } | |
| } | |
| /* | |
| instruction / input / output | |
| */ | |
| if ( | |
| typeof record.instruction === | |
| "string" && | |
| typeof record.output === | |
| "string" | |
| ) { | |
| let instruction = | |
| normalizeText( | |
| record.instruction | |
| ); | |
| const input = | |
| normalizeText( | |
| record.input || "" | |
| ); | |
| if (input) { | |
| instruction += | |
| `\n\n${input}`; | |
| } | |
| const output = | |
| normalizeText( | |
| record.output | |
| ); | |
| if ( | |
| instruction && | |
| output | |
| ) { | |
| return { | |
| messages: [ | |
| { | |
| role: "user", | |
| content: | |
| instruction | |
| }, | |
| { | |
| role: "assistant", | |
| content: | |
| output | |
| } | |
| ] | |
| }; | |
| } | |
| } | |
| /* | |
| question / answer | |
| */ | |
| if ( | |
| typeof record.question === | |
| "string" && | |
| typeof record.answer === | |
| "string" | |
| ) { | |
| const question = | |
| normalizeText( | |
| record.question | |
| ); | |
| const answer = | |
| normalizeText( | |
| record.answer | |
| ); | |
| if ( | |
| question && | |
| answer | |
| ) { | |
| return { | |
| messages: [ | |
| { | |
| role: "user", | |
| content: | |
| question | |
| }, | |
| { | |
| role: "assistant", | |
| content: | |
| answer | |
| } | |
| ] | |
| }; | |
| } | |
| } | |
| /* | |
| prompt / response | |
| */ | |
| if ( | |
| typeof record.prompt === | |
| "string" && | |
| typeof record.response === | |
| "string" | |
| ) { | |
| const prompt = | |
| normalizeText( | |
| record.prompt | |
| ); | |
| const response = | |
| normalizeText( | |
| record.response | |
| ); | |
| if ( | |
| prompt && | |
| response | |
| ) { | |
| return { | |
| messages: [ | |
| { | |
| role: "user", | |
| content: | |
| prompt | |
| }, | |
| { | |
| role: "assistant", | |
| content: | |
| response | |
| } | |
| ] | |
| }; | |
| } | |
| } | |
| /* | |
| text | |
| */ | |
| if ( | |
| typeof record.text === | |
| "string" | |
| ) { | |
| const text = | |
| normalizeText( | |
| record.text | |
| ); | |
| if (text) { | |
| return { | |
| text | |
| }; | |
| } | |
| } | |
| /* | |
| content | |
| */ | |
| if ( | |
| typeof record.content === | |
| "string" | |
| ) { | |
| const content = | |
| normalizeText( | |
| record.content | |
| ); | |
| if (content) { | |
| return { | |
| text: content | |
| }; | |
| } | |
| } | |
| /* | |
| Unknown schema: | |
| preserve it. | |
| */ | |
| return record; | |
| } | |
| /* | |
| ============================================================ | |
| QUALITY | |
| ============================================================ | |
| */ | |
| function recordText( | |
| record | |
| ) { | |
| if ( | |
| typeof record.text === | |
| "string" | |
| ) { | |
| return record.text; | |
| } | |
| if ( | |
| Array.isArray( | |
| record.messages | |
| ) | |
| ) { | |
| return record.messages | |
| .map( | |
| message => | |
| message.content || "" | |
| ) | |
| .join( | |
| "\n" | |
| ); | |
| } | |
| return JSON.stringify( | |
| record | |
| ); | |
| } | |
| function persianRatio( | |
| text | |
| ) { | |
| const letters = | |
| Array.from(text) | |
| .filter( | |
| char => | |
| /\p{L}/u.test( | |
| char | |
| ) | |
| ); | |
| if ( | |
| letters.length === 0 | |
| ) { | |
| return 0; | |
| } | |
| let persian = 0; | |
| for ( | |
| const char | |
| of letters | |
| ) { | |
| if ( | |
| /[\u0600-\u06FF\u0750-\u077F\u08A0-\u08FF]/u | |
| .test(char) | |
| ) { | |
| persian++; | |
| } | |
| } | |
| return ( | |
| persian / | |
| letters.length | |
| ); | |
| } | |
| function isSpammy( | |
| text | |
| ) { | |
| if ( | |
| text.length < 100 | |
| ) { | |
| return false; | |
| } | |
| const stripped = | |
| text.replace( | |
| /\s/g, | |
| "" | |
| ); | |
| if (!stripped) { | |
| return true; | |
| } | |
| const counts = | |
| new Map(); | |
| for ( | |
| const char | |
| of stripped | |
| ) { | |
| counts.set( | |
| char, | |
| ( | |
| counts.get( | |
| char | |
| ) || 0 | |
| ) + 1 | |
| ); | |
| } | |
| let maximum = 0; | |
| for ( | |
| const count | |
| of counts.values() | |
| ) { | |
| maximum = | |
| Math.max( | |
| maximum, | |
| count | |
| ); | |
| } | |
| if ( | |
| maximum / | |
| stripped.length | |
| >= 0.45 | |
| ) { | |
| return true; | |
| } | |
| return ( | |
| /(.{2,30})\1{5,}/s | |
| .test(text) | |
| ); | |
| } | |
| function passesQuality( | |
| record | |
| ) { | |
| const minChars = | |
| Number( | |
| $("minChars").value | |
| ) || 0; | |
| const maxChars = | |
| Number( | |
| $("maxChars").value | |
| ) || 0; | |
| const requiredRatio = | |
| Number( | |
| $("persianRatio").value | |
| ) || 0; | |
| const rejectSpam = | |
| $("rejectSpam").checked; | |
| const text = | |
| recordText( | |
| record | |
| ).trim(); | |
| if ( | |
| text.length < | |
| minChars | |
| ) { | |
| return false; | |
| } | |
| if ( | |
| maxChars > 0 && | |
| text.length > | |
| maxChars | |
| ) { | |
| return false; | |
| } | |
| if ( | |
| requiredRatio > 0 && | |
| persianRatio(text) < | |
| requiredRatio | |
| ) { | |
| return false; | |
| } | |
| if ( | |
| rejectSpam && | |
| isSpammy(text) | |
| ) { | |
| return false; | |
| } | |
| return true; | |
| } | |
| /* | |
| ============================================================ | |
| HASH / DEDUP | |
| ============================================================ | |
| */ | |
| async function hashRecord( | |
| record | |
| ) { | |
| const canonical = | |
| JSON.stringify( | |
| record, | |
| Object.keys(record) | |
| .sort() | |
| ); | |
| const data = | |
| new TextEncoder() | |
| .encode( | |
| canonical | |
| ); | |
| const digest = | |
| await crypto.subtle.digest( | |
| "SHA-256", | |
| data | |
| ); | |
| return Array.from( | |
| new Uint8Array( | |
| digest | |
| ) | |
| ) | |
| .map( | |
| byte => | |
| byte | |
| .toString(16) | |
| .padStart( | |
| 2, | |
| "0" | |
| ) | |
| ) | |
| .join(""); | |
| } | |
| /* | |
| ============================================================ | |
| DATASET STREAMING | |
| ============================================================ | |
| */ | |
| async function loadHFDataset( | |
| spec, | |
| token | |
| ) { | |
| const parts = | |
| spec.split(":"); | |
| const datasetId = | |
| parts[0]; | |
| const config = | |
| parts.length >= 2 | |
| ? parts[1] | |
| : null; | |
| const split = | |
| parts.length >= 3 | |
| ? parts[2] | |
| : "train"; | |
| /* | |
| Resolve dataset files. | |
| */ | |
| const headers = {}; | |
| if (token) { | |
| headers.Authorization = | |
| `Bearer ${token}`; | |
| } | |
| /* | |
| Get dataset metadata. | |
| */ | |
| const metadataResponse = | |
| await fetch( | |
| `${HF_API}/datasets/${encodeURIComponent(datasetId)}`, | |
| { | |
| headers | |
| } | |
| ); | |
| if ( | |
| !metadataResponse.ok | |
| ) { | |
| throw new Error( | |
| `${datasetId}: metadata HTTP ${metadataResponse.status}` | |
| ); | |
| } | |
| /* | |
| Dataset Server API. | |
| */ | |
| let rowsUrl = | |
| `https://datasets-server.huggingface.co/rows?dataset=${encodeURIComponent(datasetId)}&config=${encodeURIComponent(config || "default")}&split=${encodeURIComponent(split)}&offset=0&length=100`; | |
| const response = | |
| await fetch( | |
| rowsUrl, | |
| { | |
| headers | |
| } | |
| ); | |
| if ( | |
| !response.ok | |
| ) { | |
| throw new Error( | |
| `${datasetId}: Dataset Server HTTP ${response.status}` | |
| ); | |
| } | |
| const data = | |
| await response.json(); | |
| const rows = | |
| Array.isArray( | |
| data.rows | |
| ) | |
| ? data.rows | |
| : []; | |
| return rows.map( | |
| row => | |
| row.row || row | |
| ); | |
| } | |
| /* | |
| ============================================================ | |
| MANUAL DATASET SPECS | |
| ============================================================ | |
| */ | |
| function getDatasetSpecs() { | |
| const specs = | |
| Array.from( | |
| selectedDatasets | |
| ); | |
| const manual = | |
| manualDatasets | |
| .value | |
| .split(/\r?\n/) | |
| .map( | |
| x => x.trim() | |
| ) | |
| .filter(Boolean); | |
| for ( | |
| const spec | |
| of manual | |
| ) { | |
| if ( | |
| !specs.includes( | |
| spec | |
| ) | |
| ) { | |
| specs.push( | |
| spec | |
| ); | |
| } | |
| } | |
| return specs; | |
| } | |
| /* | |
| ============================================================ | |
| WEIGHTS | |
| ============================================================ | |
| */ | |
| function getWeights() { | |
| const weights = {}; | |
| const lines = | |
| $("weights") | |
| .value | |
| .split(/\r?\n/); | |
| for ( | |
| const line | |
| of lines | |
| ) { | |
| if ( | |
| !line.includes("=") | |
| ) { | |
| continue; | |
| } | |
| const [ | |
| name, | |
| raw | |
| ] = | |
| line.split( | |
| "=", | |
| 2 | |
| ); | |
| const value = | |
| Number( | |
| raw.trim() | |
| ); | |
| if ( | |
| Number.isFinite(value) | |
| ) { | |
| weights[ | |
| name.trim() | |
| ] = | |
| Math.max( | |
| 0, | |
| Math.min( | |
| 1, | |
| value | |
| ) | |
| ); | |
| } | |
| } | |
| return weights; | |
| } | |
| /* | |
| ============================================================ | |
| SHUFFLE | |
| ============================================================ | |
| */ | |
| function seededRandom( | |
| seed | |
| ) { | |
| let value = | |
| Number(seed) || 42; | |
| return function () { | |
| value = | |
| ( | |
| value * 1664525 | |
| + 1013904223 | |
| ) % 4294967296; | |
| return ( | |
| value / | |
| 4294967296 | |
| ); | |
| }; | |
| } | |
| function shuffleArray( | |
| array, | |
| seed | |
| ) { | |
| const random = | |
| seededRandom( | |
| seed | |
| ); | |
| for ( | |
| let i = | |
| array.length - 1; | |
| i > 0; | |
| i-- | |
| ) { | |
| const j = | |
| Math.floor( | |
| random() | |
| * | |
| (i + 1) | |
| ); | |
| [ | |
| array[i], | |
| array[j] | |
| ] = [ | |
| array[j], | |
| array[i] | |
| ]; | |
| } | |
| } | |
| /* | |
| ============================================================ | |
| JSONL | |
| ============================================================ | |
| */ | |
| function recordsToJSONL( | |
| records | |
| ) { | |
| return records | |
| .map( | |
| record => | |
| JSON.stringify( | |
| record | |
| ) | |
| ) | |
| .join( | |
| "\n" | |
| ) | |
| + "\n"; | |
| } | |
| /* | |
| ============================================================ | |
| BUILD | |
| ============================================================ | |
| */ | |
| buildButton.addEventListener( | |
| "click", | |
| buildDataset | |
| ); | |
| async function buildDataset() { | |
| const token = | |
| hfToken.value.trim(); | |
| const repoName = | |
| $("repoName") | |
| .value | |
| .trim(); | |
| if (!token) { | |
| alert( | |
| "Hugging Face Token را وارد کنید." | |
| ); | |
| return; | |
| } | |
| if (!repoName) { | |
| alert( | |
| "نام Repository را وارد کنید." | |
| ); | |
| return; | |
| } | |
| if ( | |
| selectedDatasets.size === 0 && | |
| localFiles.length === 0 && | |
| !manualDatasets.value.trim() | |
| ) { | |
| alert( | |
| "حداقل یک دیتاست یا فایل اضافه کنید." | |
| ); | |
| return; | |
| } | |
| clearLog(); | |
| progressSection.classList.remove( | |
| "hidden" | |
| ); | |
| resultBox.classList.add( | |
| "hidden" | |
| ); | |
| buildButton.disabled = | |
| true; | |
| generatedTrainBlob = | |
| null; | |
| try { | |
| log( | |
| "🚀 شروع پردازش..." | |
| ); | |
| const allRecords = []; | |
| const deduplicate = | |
| $("deduplicate").checked; | |
| const maxRecords = | |
| Number( | |
| $("maxRecords").value | |
| ) || 0; | |
| const weights = | |
| getWeights(); | |
| const specs = | |
| getDatasetSpecs(); | |
| const seen = | |
| new Set(); | |
| /* | |
| ==================================================== | |
| LOCAL FILES | |
| ==================================================== | |
| */ | |
| for ( | |
| let i = 0; | |
| i < localFiles.length; | |
| i++ | |
| ) { | |
| const file = | |
| localFiles[i]; | |
| setProgress( | |
| 5 + | |
| ( | |
| i / | |
| Math.max( | |
| localFiles.length, | |
| 1 | |
| ) | |
| ) | |
| * 20, | |
| `خواندن ${file.name}` | |
| ); | |
| log( | |
| `📁 ${file.name}` | |
| ); | |
| const records = | |
| await parseFile( | |
| file | |
| ); | |
| log( | |
| ` ${records.length.toLocaleString()} رکورد` | |
| ); | |
| for ( | |
| const raw | |
| of records | |
| ) { | |
| if ( | |
| maxRecords > 0 && | |
| allRecords.length >= | |
| maxRecords | |
| ) { | |
| break; | |
| } | |
| const record = | |
| normalizeRecord( | |
| raw | |
| ); | |
| if (!record) { | |
| continue; | |
| } | |
| if ( | |
| !passesQuality( | |
| record | |
| ) | |
| ) { | |
| continue; | |
| } | |
| if ( | |
| deduplicate | |
| ) { | |
| const hash = | |
| await hashRecord( | |
| record | |
| ); | |
| if ( | |
| seen.has( | |
| hash | |
| ) | |
| ) { | |
| continue; | |
| } | |
| seen.add( | |
| hash | |
| ); | |
| } | |
| allRecords.push( | |
| record | |
| ); | |
| } | |
| } | |
| /* | |
| ==================================================== | |
| HF DATASETS | |
| ==================================================== | |
| */ | |
| for ( | |
| let i = 0; | |
| i < specs.length; | |
| i++ | |
| ) { | |
| if ( | |
| maxRecords > 0 && | |
| allRecords.length >= | |
| maxRecords | |
| ) { | |
| break; | |
| } | |
| const spec = | |
| specs[i]; | |
| const datasetId = | |
| spec.split(":")[0]; | |
| const weight = | |
| weights[spec] ?? | |
| weights[datasetId] ?? | |
| 1; | |
| if ( | |
| weight <= 0 | |
| ) { | |
| log( | |
| `⏭️ ${spec} — weight=0` | |
| ); | |
| continue; | |
| } | |
| setProgress( | |
| 25 + | |
| ( | |
| i / | |
| Math.max( | |
| specs.length, | |
| 1 | |
| ) | |
| ) | |
| * 45, | |
| `دریافت ${spec}` | |
| ); | |
| log( | |
| `🔗 ${spec}` | |
| ); | |
| try { | |
| const rows = | |
| await loadHFDataset( | |
| spec, | |
| token | |
| ); | |
| log( | |
| ` دریافت ${rows.length.toLocaleString()} رکورد` | |
| ); | |
| const random = | |
| Math.random; | |
| for ( | |
| const raw | |
| of rows | |
| ) { | |
| if ( | |
| maxRecords > 0 && | |
| allRecords.length >= | |
| maxRecords | |
| ) { | |
| break; | |
| } | |
| if ( | |
| weight < 1 && | |
| random() > | |
| weight | |
| ) { | |
| continue; | |
| } | |
| const record = | |
| normalizeRecord( | |
| raw | |
| ); | |
| if (!record) { | |
| continue; | |
| } | |
| if ( | |
| !passesQuality( | |
| record | |
| ) | |
| ) { | |
| continue; | |
| } | |
| if ( | |
| deduplicate | |
| ) { | |
| const hash = | |
| await hashRecord( | |
| record | |
| ); | |
| if ( | |
| seen.has( | |
| hash | |
| ) | |
| ) { | |
| continue; | |
| } | |
| seen.add( | |
| hash | |
| ); | |
| } | |
| allRecords.push( | |
| record | |
| ); | |
| } | |
| } catch (error) { | |
| log( | |
| ` ❌ ${error.message}` | |
| ); | |
| } | |
| } | |
| /* | |
| ==================================================== | |
| EMPTY | |
| ==================================================== | |
| */ | |
| if ( | |
| allRecords.length === 0 | |
| ) { | |
| throw new Error( | |
| "هیچ رکورد قابل استفادهای پیدا نشد." | |
| ); | |
| } | |
| /* | |
| ==================================================== | |
| SHUFFLE | |
| ==================================================== | |
| */ | |
| setProgress( | |
| 75, | |
| "Shuffle..." | |
| ); | |
| if ( | |
| $("shuffle").checked | |
| ) { | |
| shuffleArray( | |
| allRecords, | |
| Number( | |
| $("seed").value | |
| ) || 42 | |
| ); | |
| } | |
| /* | |
| ==================================================== | |
| VALIDATION | |
| ==================================================== | |
| */ | |
| const validationPercent = | |
| Math.max( | |
| 0, | |
| Math.min( | |
| 50, | |
| Number( | |
| $("validation").value | |
| ) || 0 | |
| ) | |
| ); | |
| const validationCount = | |
| Math.floor( | |
| allRecords.length | |
| * | |
| validationPercent | |
| / | |
| 100 | |
| ); | |
| const validationRecords = | |
| allRecords.splice( | |
| 0, | |
| validationCount | |
| ); | |
| /* | |
| ==================================================== | |
| TRAIN JSONL | |
| ==================================================== | |
| */ | |
| setProgress( | |
| 82, | |
| "ساخت train.jsonl..." | |
| ); | |
| const trainJSONL = | |
| recordsToJSONL( | |
| allRecords | |
| ); | |
| const trainBlob = | |
| new Blob( | |
| [ | |
| trainJSONL | |
| ], | |
| { | |
| type: | |
| "application/jsonl" | |
| } | |
| ); | |
| generatedTrainBlob = | |
| trainBlob; | |
| /* | |
| ==================================================== | |
| DOWNLOAD URL | |
| ==================================================== | |
| */ | |
| if ( | |
| generatedTrainUrl | |
| ) { | |
| URL.revokeObjectURL( | |
| generatedTrainUrl | |
| ); | |
| } | |
| generatedTrainUrl = | |
| URL.createObjectURL( | |
| trainBlob | |
| ); | |
| downloadTrain.onclick = | |
| () => { | |
| const link = | |
| document.createElement( | |
| "a" | |
| ); | |
| link.href = | |
| generatedTrainUrl; | |
| link.download = | |
| "train.jsonl"; | |
| link.click(); | |
| }; | |
| /* | |
| ==================================================== | |
| UPLOAD | |
| ==================================================== | |
| */ | |
| setProgress( | |
| 88, | |
| "ساخت Repository..." | |
| ); | |
| const repo = | |
| normalizeRepoName( | |
| repoName | |
| ); | |
| const visibility = | |
| $("visibility").value; | |
| await createRepo( | |
| repo, | |
| visibility, | |
| token | |
| ); | |
| log( | |
| `📦 Repository: ${repo}` | |
| ); | |
| setProgress( | |
| 93, | |
| "آپلود train.jsonl..." | |
| ); | |
| await uploadFile( | |
| repo, | |
| "train.jsonl", | |
| trainBlob, | |
| token | |
| ); | |
| /* | |
| validation | |
| */ | |
| if ( | |
| validationRecords.length | |
| ) { | |
| const validationBlob = | |
| new Blob( | |
| [ | |
| recordsToJSONL( | |
| validationRecords | |
| ) | |
| ], | |
| { | |
| type: | |
| "application/jsonl" | |
| } | |
| ); | |
| await uploadFile( | |
| repo, | |
| "validation.jsonl", | |
| validationBlob, | |
| token | |
| ); | |
| log( | |
| `🧪 validation.jsonl: ${validationRecords.length.toLocaleString()} رکورد` | |
| ); | |
| } | |
| /* | |
| ==================================================== | |
| DONE | |
| ==================================================== | |
| */ | |
| generatedRepoUrl = | |
| `https://huggingface.co/datasets/${repo}`; | |
| repoButton.onclick = | |
| () => { | |
| window.open( | |
| generatedRepoUrl, | |
| "_blank" | |
| ); | |
| }; | |
| setProgress( | |
| 100, | |
| "تمام شد!" | |
| ); | |
| log(""); | |
| log( | |
| "━━━━━━━━━━━━━━━━━━━━━━━━━━━━" | |
| ); | |
| log( | |
| "✅ DATASET COMPLETED" | |
| ); | |
| log( | |
| "━━━━━━━━━━━━━━━━━━━━━━━━━━━━" | |
| ); | |
| log( | |
| `🚂 Train: ${allRecords.length.toLocaleString()}` | |
| ); | |
| log( | |
| `🧪 Validation: ${validationRecords.length.toLocaleString()}` | |
| ); | |
| log( | |
| `📦 Size: ${formatBytes(trainBlob.size)}` | |
| ); | |
| log( | |
| `🔗 ${generatedRepoUrl}` | |
| ); | |
| resultInfo.textContent = | |
| `${allRecords.length.toLocaleString()} رکورد در train.jsonl ساخته شد.`; | |
| resultBox.classList.remove( | |
| "hidden" | |
| ); | |
| } catch (error) { | |
| log( | |
| `❌ ${error.message}` | |
| ); | |
| alert( | |
| error.message | |
| ); | |
| } finally { | |
| buildButton.disabled = | |
| false; | |
| } | |
| } | |
| /* | |
| ============================================================ | |
| REPOSITORY | |
| ============================================================ | |
| */ | |
| function normalizeRepoName( | |
| name | |
| ) { | |
| name = | |
| name | |
| .replace( | |
| /^https?:\/\/huggingface\.co\/datasets\//, | |
| "" | |
| ) | |
| .replace( | |
| /^\/+/, | |
| "" | |
| ) | |
| .trim(); | |
| return name; | |
| } | |
| async function createRepo( | |
| repo, | |
| visibility, | |
| token | |
| ) { | |
| const response = | |
| await fetch( | |
| `${HF_API}/repos/create`, | |
| { | |
| method: | |
| "POST", | |
| headers: { | |
| Authorization: | |
| `Bearer ${token}`, | |
| "Content-Type": | |
| "application/json" | |
| }, | |
| body: | |
| JSON.stringify( | |
| { | |
| name: | |
| repo.split( | |
| "/" | |
| ).pop(), | |
| organization: | |
| repo.includes( | |
| "/" | |
| ) | |
| ? repo.split( | |
| "/" | |
| )[0] | |
| : undefined, | |
| private: | |
| visibility === | |
| "private", | |
| type: | |
| "dataset" | |
| } | |
| ) | |
| } | |
| ); | |
| if ( | |
| !response.ok && | |
| response.status !== 409 | |
| ) { | |
| const text = | |
| await response.text(); | |
| throw new Error( | |
| `Repository creation failed: ${response.status} ${text}` | |
| ); | |
| } | |
| } | |
| /* | |
| ============================================================ | |
| UPLOAD FILE | |
| ============================================================ | |
| */ | |
| async function uploadFile( | |
| repo, | |
| path, | |
| blob, | |
| token | |
| ) { | |
| /* | |
| Hugging Face Hub commit API. | |
| */ | |
| const [owner, name] = | |
| repo.split( | |
| "/" | |
| ); | |
| if ( | |
| !owner || | |
| !name | |
| ) { | |
| throw new Error( | |
| "Repository باید به شکل username/name باشد." | |
| ); | |
| } | |
| const form = | |
| new FormData(); | |
| form.append( | |
| "file", | |
| blob, | |
| path | |
| ); | |
| const response = | |
| await fetch( | |
| `https://huggingface.co/api/datasets/${encodeURIComponent(owner)}/${encodeURIComponent(name)}/upload`, | |
| { | |
| method: | |
| "POST", | |
| headers: { | |
| Authorization: | |
| `Bearer ${token}` | |
| }, | |
| body: | |
| form | |
| } | |
| ); | |
| if ( | |
| !response.ok | |
| ) { | |
| const text = | |
| await response.text(); | |
| throw new Error( | |
| `Upload failed: ${response.status} ${text}` | |
| ); | |
| } | |
| } |