diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..c72c60f6c08ffa884b5b75b07a58c510c1820b0d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,12 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +# Image files - uncompressed +*.bmp filter=lfs diff=lfs merge=lfs -text +*.gif filter=lfs diff=lfs merge=lfs -text +*.png filter=lfs diff=lfs merge=lfs -text +*.tiff filter=lfs diff=lfs merge=lfs -text +# Image files - compressed +*.jpg filter=lfs diff=lfs merge=lfs -text +*.jpeg filter=lfs diff=lfs merge=lfs -text +*.webp filter=lfs diff=lfs merge=lfs -text \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..008f7fdfd37e0e5e694ea8e807be4f1e4aecaff4 --- /dev/null +++ b/README.md @@ -0,0 +1,178 @@ +--- +license: apache-2.0 +library_name: pytorch +tags: +- code +- classification +- code-filtering +- algorithmic-relevance +- ultradata +language: +- en +- zh +--- + +# UltraData-Code-L2-Classifier + +

+📦 UltraData Collection | +🌐 UltraData | +🤗 MiniCPM5 Series | +📖 Tech Report (Coming Soon) | +🤗 UltraData-Code Dataset +

+ +

English | 中文

+ +## 📚 Introduction + +***UltraData-Code-L2-Classifier*** is a suite of language-specific, file-level scorers for the 11 programming languages covered by [UltraData-Code](https://huggingface.co/datasets/openbmb/UltraData-Code). It is the L2 selection model used to build **UltraData-Code-L2** from UltraData-Code-L1. + +For each language, a dedicated scorer ranks L1 files by algorithmic relevance and code quality. Selected files form UltraData-Code-L2 (~**400B tokens**), retaining about **12.23%** of L1 files overall. The goal is to concentrate interpretable computation and problem-solving logic for code pretraining. + +## 📢 What's New + +- **[2026.09.07]** ***UltraData-Code-L2-Classifier*** is released, together with [***UltraData-Code***](https://huggingface.co/datasets/openbmb/UltraData-Code) (**L2** \~400B tokens / **L3** \~150B tokens across 11 languages). 🚀🚀🚀 +- **[2026.09.07]** ***[MiniCPM5-2B](https://huggingface.co/openbmb/MiniCPM5-2B) is released!***, the second model in the MiniCPM5 series after MiniCPM5-1B. 🚀🚀🚀 +- **[2026.02.08]** The [***UltraData***](https://ultradata.openbmb.cn/) platform is now live, introducing the [L0-L4 tiered data management framework](https://arxiv.org/pdf/2602.09003). 🔍🔍🔍 + +## 💡 Highlights + +> **Abstract:** Code generation is a core capability of large language models, and code data is central to the pretraining process that develops it. As code corpora grow, their scale, diversity, and quality increasingly shape the capabilities learned during pretraining. Following a tiered data management perspective, we present **UltraData-Code** as a family of four connected data states: repository archival at L0, standardized natural code at L1, algorithmic selection at L2, and task-oriented synthesis at L3. L0 archives the latest revision on the default branch of each public GitHub repository with its file structure, relationships, and provenance. L1 applies scalable filtering, normalization, and near deduplication to obtain standardized natural code. L2 selects algorithmically relevant files from L1 using language-adaptive signals from file roles and heuristics, together with code-quality constraints, yielding approximately 400B tokens across 11 programming languages. L3 applies task-oriented synthesis to algorithmic files from L2, turning each implementation into a structured programming exercise with aligned task, analysis, solution, and test fields, and generating approximately 150B tokens in the same 11 languages. Under controlled 10B-token continual pretraining of a 1B model, training on L2 instead of L1 raises pass@1 on EvalPlus by **7.80 points** and on MultiPL-E by **5.13 points**, while exceeding Stack-Edu by 4.37 and 3.05 points, respectively. Replacing half of the L2 training tokens with L3 yields a further gain of **8.42 points** on EvalPlus and **8.07 points** on MultiPL-E over training on L2 alone, while exceeding the strongest synthetic data baseline by 5.57 and 7.80 points, respectively. + +- **A connected code data construction pipeline:** UltraData-Code links repository archival and standardized natural code with fine-grained selection and structured synthesis, yielding approximately 400B tokens at L2 and 150B tokens at L3 across 11 programming languages. +- **Fine-grained curation of algorithmically relevant code:** We introduce file-level algorithmic relevance as a selection objective and combine file-role cues with language-specific heuristic evidence to identify computational content beyond files explicitly labeled as algorithm implementations. +- **Task-oriented synthesis grounded in implementations:** Each selected implementation serves as the common source for a task, analysis, solution, and test record, preserving computational intent while adding task semantics and structured supervision. + +
+ UltraData-Code L0-L3 construction pipeline +
+ +## 🔎 L2 Selection Framework + +The L2 framework is applied independently to each of the 11 languages in UltraData-Code-L1: + +1. **Encode files.** Each file is represented with a 1024-dimensional embedding from [Qwen3-Embedding-0.6B](https://huggingface.co/Qwen/Qwen3-Embedding-0.6B). +2. **Predict file roles.** The role classifier assigns one of `ALGO`, `WEB`, `TOOL`, `DATA`, `TEST`, `CONFIG`, or `EXCLUDE`. +3. **Construct dual-cue supervision.** ALGO role labels are combined with language-specific heuristic evidence to identify algorithmically relevant files beyond the ALGO role alone. +4. **Score relevance and quality.** The relevance model outputs `algo_rel_score`; the quality model outputs `quality_score`. +5. **Calibrate and select.** A language-specific relevance threshold is calibrated on held-out data. A file is retained when it passes the relevance and quality thresholds and is not predicted as `EXCLUDE`. + +
+ UltraData-Code L2 selection framework +
+ +## 📈 Evaluation Results + +The 10B-token comparisons use the same 1B base model, architecture, optimization settings, preprocessing, decontamination, and evaluation protocol. + +### Python Results + +- **Algorithmic selection yields the strongest natural code result.** UltraData-Code-L2-py improves the average EvalPlus pass@1 by **17.91 points** over UltraData-Code-L1-py and by **8.46 points** over Stack-Edu-py. Ultra-Stack-v3-py reaches the second-highest natural code average, while UltraData-Code-L2-py records the strongest average overall among the natural code settings. +- **Structured synthesis improves the matched mixture.** The equal-token UltraData-Code-L2-py + UltraData-Code-L3-py mixture reaches a **46.43** average pass@1, **3.30 points** above training on L2 alone and **4.46 points** above the matched L2 + SwallowCode-v2 mixture. + +
+ Python training dynamics +
+ +
+ Python main results table +
+ +### Multilingual Results + +- **L2 selection improves over L1 baselines.** UltraData-Code-L2 exceeds UltraData-Code-L1 by **7.80 points** on EvalPlus and **5.13 points** on MultiPL-E, and exceeds Stack-Edu by **4.37** and **3.05 points**, respectively. +- **L3 synthesis provides a further gain.** Replacing half of the L2 training tokens with L3 yields **8.42** and **8.07 points** over training on L2 alone on EvalPlus and MultiPL-E. Under the same 1:1 token mix, UltraData-Code-L2-L3 exceeds the strongest reported synthetic baseline by **5.57** and **7.80 points**, respectively. +- **The advantage persists during scaling.** At 100B tokens, the L2-L3 mixture reaches 57.06 on EvalPlus and 39.54 on MultiPL-E, exceeding training on L2 alone by 10.11 and 12.39 points and the matched CodeAlchemy-code-qa mixture by 10.86 and 9.60 points. + +
+ Multilingual training dynamics +
+ +
+ Multilingual main results table +
+ +## 🚀 Usage + +```bash +pip install numpy torch pyarrow +``` + +### Score embeddings in Python + +```python +import sys +sys.path.insert(0, "scripts") +from scorers import LanguageScorers + +scorers = LanguageScorers.load("classifiers/cpp") + +# float32 [N, 1024]; paths are repository-relative and aligned with embeddings +out = scorers.score(embeddings, paths) +``` + +Use the matching language directory under `classifiers/` (`cpp`, `csharp`, `go`, `java`, `javascript`, `php`, `python`, `r`, `ruby`, `rust`, `shell`). Real repository-relative paths are required by the relevance model. + +### Score Parquet files + +[`scripts/score_parquet.py`](scripts/score_parquet.py) accepts one Parquet file or recursively walks a directory. Inputs must contain an `embedding` column and either `relative_path` or `meta.file_path`. + +```bash +SCORERS=classifiers/cpp +DATA_PATH=data/input +SAVE_PATH=data/output + +python scripts/score_parquet.py \ + --scorers ${SCORERS} \ + --input ${DATA_PATH} \ + --output-root ${SAVE_PATH} \ + --batch-size 1024 \ + --device cpu \ + --apply-policy +``` + +The output mirrors the input directory structure and keeps existing columns. Use `--describe` to inspect a scorer set, `--limit N` to process at most `N` Parquet files, or `--algo-rel-min`, `--quality-min`, and `--exclude-categories` to override policy values. + +### Output fields + +```text +category: {Predicted file role} +cls_confidence: {File-role confidence} +algo_rel_score: {Algorithmic-relevance score} +quality_score: {Code-quality score} +selected: {Whether the file passes the selection policy} +``` + +- `category` ∈ {`ALGO`, `WEB`, `TOOL`, `DATA`, `TEST`, `CONFIG`, `EXCLUDE`} +- `cls_confidence` / `algo_rel_score` ∈ `[0, 1]`; `quality_score` ∈ `[0, 10]` +- `selected` is written only with `--apply-policy`; it marks policy passes and does not delete rows + +### Notes + +- Embeddings must be 1024-d from [Qwen3-Embedding-0.6B](https://huggingface.co/Qwen/Qwen3-Embedding-0.6B). +- The relevance scorer also uses the repository-relative path; keep paths aligned with embeddings. +- [`config.json`](config.json) defaults: `algo_rel_min = 0.8`, `quality_min = 3.0`, veto category `EXCLUDE`. CLI flags override these. +- `score_parquet.py` is single-process; shard externally for large corpora. + +## ❤️ Acknowledgements + +This release builds on [Qwen3-Embedding-0.6B](https://huggingface.co/Qwen/Qwen3-Embedding-0.6B), [PyTorch](https://pytorch.org/), and [PyArrow](https://arrow.apache.org/docs/python/). Thanks for their awesome work! 🙌 + +## 💳 License + +This project is released under the [Apache 2.0](https://www.apache.org/licenses/LICENSE-2.0) license. + +## 📖 Citation + +If you find **UltraData-Code** or this classifier useful, please consider citing: + +```bibtex +@misc{ultradata_code, + title = {{UltraData-Code}: From Raw Repositories to Algorithmically Dense and Task-Oriented Code Data}, + author = {Chengying Tu and Hengyu Zhao and Shuaikang Xue and Zhongming Qu and Jihao Zhou and Xinle Lin and Junshao Guo and Zixuan Fu and Qiang Ma and Jie Zhou and Chaojun Xiao and Hongfei Yan and Yudong Wang and Xu Han and Zhiyuan Liu and Maosong Sun}, + year = {2026}, + publisher = {Hugging Face}, + howpublished = {\url{https://huggingface.co/datasets/openbmb/UltraData-Code}} +} +``` diff --git a/README_ZH.md b/README_ZH.md new file mode 100644 index 0000000000000000000000000000000000000000..335102e6d090cd1aa5e09c51534278ac65218524 --- /dev/null +++ b/README_ZH.md @@ -0,0 +1,164 @@ +# UltraData-Code-L2-Classifier + +

+📦 UltraData 合集 | +🌐 UltraData | +🤗 MiniCPM5 系列 | +📖 技术报告（即将发布）| +🤗 UltraData-Code 数据集 +

+ +

English | 中文

+ +## 📚 简介 + +***UltraData-Code-L2-Classifier*** 是面向 [UltraData-Code](https://huggingface.co/datasets/openbmb/UltraData-Code) 覆盖的 11 种编程语言的一组语言特定、文件级打分器。它是从 UltraData-Code-L1 构建 **UltraData-Code-L2** 所使用的 L2 筛选模型。 + +对每种语言，专用打分器按算法相关性和代码质量对 L1 文件排序。筛选后的文件构成 UltraData-Code-L2（约 **400B tokens**），总体保留 L1 约 **12.23%** 的文件。目标是为代码预训练集中可解释的计算与问题求解逻辑。 + +## 📢 最新动态 + +- **[2026.09.07]** ***UltraData-Code-L2-Classifier*** 正式发布，同步开源 [***UltraData-Code***](https://huggingface.co/datasets/openbmb/UltraData-Code)（**L2** 约 400B tokens / **L3** 约 150B tokens，覆盖 11 种语言）。🚀🚀🚀 +- **[2026.09.07]** ***[MiniCPM5-2B](https://huggingface.co/openbmb/MiniCPM5-2B) 正式发布！*** 作为 MiniCPM5 系列在 MiniCPM5-1B 之后的第二款模型。🚀🚀🚀 +- **[2026.02.08]** [***UltraData***](https://ultradata.openbmb.cn/) 平台正式上线，并发布 [L0-L4 分级数据管理框架](https://arxiv.org/pdf/2602.09003)。🔍🔍🔍 + +## 💡 亮点 + +> **摘要：** 代码生成是大语言模型的核心能力，代码数据是培养该能力的预训练过程的重要组成部分。随着代码语料规模不断增长，其规模、多样性和质量日益决定模型在预训练中学习到的能力。遵循分级数据治理视角，我们将 **UltraData-Code** 构建为四个相互连接的数据状态：L0 仓库归档、L1 规范化自然代码、L2 算法相关性精选和 L3 面向任务的合成。L0 保留每个公开 GitHub 仓库默认分支的最新版本及其文件结构、文件关系和来源信息。L1 通过规模化过滤、规范化和近重复去重得到标准化自然代码。L2 基于文件角色、语言自适应启发式信号和代码质量约束，从 L1 中筛选算法相关文件，覆盖 11 种编程语言、约 400B tokens。L3 以 L2 中的算法相关文件为基础，将每个实现转换为任务、分析、解答和测试字段对齐的结构化编程练习，覆盖相同的 11 种语言、约 150B tokens。在相同的 1B 模型和 10B-token 持续预训练设置下，使用 L2 替代 L1 后，EvalPlus 的 pass@1 提升 **7.80 个百分点**，MultiPL-E 提升 **5.13 个百分点**，相比 Stack-Edu 分别提升 4.37 和 3.05 个百分点。将一半 L2 训练 token 替换为 L3 后，相比仅使用 L2 的训练，EvalPlus 进一步提升 **8.42 个百分点**，MultiPL-E 进一步提升 **8.07 个百分点**，相比最强的合成数据基线分别提升 5.57 和 7.80 个百分点。 + +- **完整的代码数据构建流水线：** 将仓库归档和规范化自然代码与细粒度精选、结构化合成连接起来，在 11 种编程语言上分别形成约 400B tokens 的 L2 和约 150B tokens 的 L3。 +- **细粒度算法相关代码精选：** 将文件级算法相关性定义为选择目标，结合文件角色线索与语言特定启发式证据，识别超出显式算法实现标签范围的计算内容。 +- **面向任务且以实现为基础的合成：** 以每个精选实现作为任务、分析、解答和测试记录的共同来源，在保留计算意图的同时补充任务语义和结构化监督。 + +
+ UltraData-Code L0-L3 构建流程 +
+ +## 🔎 L2 筛选框架 + +L2 框架分别独立应用于 UltraData-Code-L1 中的 11 种语言： + +1. **编码文件。** 使用 [Qwen3-Embedding-0.6B](https://huggingface.co/Qwen/Qwen3-Embedding-0.6B) 为每个文件生成 1024 维 embedding。 +2. **预测文件角色。** 角色分类器将文件分为 `ALGO`、`WEB`、`TOOL`、`DATA`、`TEST`、`CONFIG` 或 `EXCLUDE`。 +3. **构造双线索监督。** 将 ALGO 角色标注与语言特定启发式证据结合，识别 ALGO 角色之外的算法相关文件。 +4. **打分相关性与质量。** 相关性模型输出 `algo_rel_score`，质量模型输出 `quality_score`。 +5. **校准并筛选。** 在留出数据上为每种语言校准相关性阈值。文件需通过相关性和质量阈值，且未被预测为 `EXCLUDE`，才会保留。 + +
+ UltraData-Code L2 筛选框架 +
+ +## 📈 评测结果 + +10B-token 对比均使用相同的 1B 基座模型、模型架构、优化设置、预处理、去污染流程和评测协议。 + +### Python 结果 + +- **算法相关性精选带来最强的自然代码结果。** UltraData-Code-L2-py 的 EvalPlus 平均 pass@1 比 UltraData-Code-L1-py 提升 **17.91 个百分点**，比 Stack-Edu-py 提升 **8.46 个百分点**。Ultra-Stack-v3-py 在自然代码设置中取得第二高的平均分，而 UltraData-Code-L2-py 在自然代码设置中取得最高平均分。 +- **结构化合成提升等 token 混合训练效果。** UltraData-Code-L2-py 与 UltraData-Code-L3-py 的等 token 混合训练平均 pass@1 达到 **46.43**，比仅使用 L2 高 **3.30 个百分点**，比对应的 L2 + SwallowCode-v2 混合训练高 **4.46 个百分点**。 + +
+ Python 训练动态 +
+ +
+ Python 主结果表 +
+ +### 多语言结果 + +- **L2 精选优于自然代码基线。** UltraData-Code-L2 相比 UltraData-Code-L1 在 EvalPlus 和 MultiPL-E 上分别提升 **7.80** 和 **5.13 个百分点**，相比 Stack-Edu 分别提升 **4.37** 和 **3.05 个百分点**。 +- **L3 合成带来进一步提升。** 将一半 L2 训练 token 替换为 L3 后，相比仅使用 L2 的训练，EvalPlus 和 MultiPL-E 分别提升 **8.42** 和 **8.07 个百分点**。在相同的 1:1 token 配比下，UltraData-Code-L2-L3 混合训练相比最强的已报告合成数据基线，两个基准分别提升 **5.57** 和 **7.80 个百分点**。 +- **优势在规模扩展中持续存在。** 在 100B tokens 训练规模下，L2-L3 混合训练在 EvalPlus 和 MultiPL-E 上分别达到 57.06 和 39.54，相比仅使用 L2 的训练分别高 10.11 和 12.39 个百分点，相比对应的 CodeAlchemy-code-qa 混合训练分别高 10.86 和 9.60 个百分点。 + +
+ 多语言训练动态 +
+ +
+ 多语言主结果表 +
+ +## 🚀 使用方法 + +```bash +pip install numpy torch pyarrow +``` + +### 在 Python 中对 embedding 打分 + +```python +import sys +sys.path.insert(0, "scripts") +from scorers import LanguageScorers + +scorers = LanguageScorers.load("classifiers/cpp") + +# float32 [N, 1024]；paths 为仓库相对路径，并与 embeddings 对齐 +out = scorers.score(embeddings, paths) +``` + +请使用 `classifiers/` 下对应语言目录（`cpp`、`csharp`、`go`、`java`、`javascript`、`php`、`python`、`r`、`ruby`、`rust`、`shell`）。相关性模型需要真实的仓库相对路径。 + +### 对 Parquet 文件打分 + +[`scripts/score_parquet.py`](scripts/score_parquet.py) 可处理单个 Parquet 文件，或递归处理目录。输入必须包含 `embedding` 列，以及 `relative_path` 或 `meta.file_path`。 + +```bash +SCORERS=classifiers/cpp +DATA_PATH=data/input +SAVE_PATH=data/output + +python scripts/score_parquet.py \ + --scorers ${SCORERS} \ + --input ${DATA_PATH} \ + --output-root ${SAVE_PATH} \ + --batch-size 1024 \ + --device cpu \ + --apply-policy +``` + +输出会镜像输入目录结构，并保留原有列。使用 `--describe` 查看打分器配置，使用 `--limit N` 最多处理 `N` 个 Parquet 文件，或使用 `--algo-rel-min`、`--quality-min` 和 `--exclude-categories` 覆盖策略值。 + +### 输出字段 + +```text +category: {预测的文件角色} +cls_confidence: {文件角色置信度} +algo_rel_score: {算法相关性分数} +quality_score: {代码质量分数} +selected: {是否通过筛选策略} +``` + +- `category` ∈ {`ALGO`, `WEB`, `TOOL`, `DATA`, `TEST`, `CONFIG`, `EXCLUDE`} +- `cls_confidence` / `algo_rel_score` ∈ `[0, 1]`；`quality_score` ∈ `[0, 10]` +- 仅在启用 `--apply-policy` 时写入 `selected`；它标记通过策略的行，不会删除未通过的行 + +### 注意事项 + +- embedding 必须来自 [Qwen3-Embedding-0.6B](https://huggingface.co/Qwen/Qwen3-Embedding-0.6B) 的 1024 维表示。 +- 相关性打分器还会使用仓库相对路径；请保持路径与 embedding 对齐。 +- [`config.json`](config.json) 默认值：`algo_rel_min = 0.8`、`quality_min = 3.0`，veto 类别为 `EXCLUDE`。命令行参数优先。 +- `score_parquet.py` 为单进程实现；大规模语料请在外部分片调度。 + +## ❤️ 致谢 + +本发布基于 [Qwen3-Embedding-0.6B](https://huggingface.co/Qwen/Qwen3-Embedding-0.6B)、[PyTorch](https://pytorch.org/) 和 [PyArrow](https://arrow.apache.org/docs/python/)。感谢这些优秀开源工作！🙌 + +## 💳 许可证 + +本项目基于 [Apache 2.0](https://www.apache.org/licenses/LICENSE-2.0) 许可证发布。 + +## 📖 引用 + +如果 **UltraData-Code** 或本 Classifier 对您的研究有帮助，请考虑引用： + +```bibtex +@misc{ultradata_code, + title = {{UltraData-Code}: From Raw Repositories to Algorithmically Dense and Task-Oriented Code Data}, + author = {Chengying Tu and Hengyu Zhao and Shuaikang Xue and Zhongming Qu and Jihao Zhou and Xinle Lin and Junshao Guo and Zixuan Fu and Qiang Ma and Jie Zhou and Chaojun Xiao and Hongfei Yan and Yudong Wang and Xu Han and Zhiyuan Liu and Maosong Sun}, + year = {2026}, + publisher = {Hugging Face}, + howpublished = {\url{https://huggingface.co/datasets/openbmb/UltraData-Code}} +} +``` diff --git a/assets/multilingual-training-dynamics.png b/assets/multilingual-training-dynamics.png new file mode 100644 index 0000000000000000000000000000000000000000..9bf543b0aa78414f95ebb3dd770b14dea52988cd --- /dev/null +++ b/assets/multilingual-training-dynamics.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66d7b047effac3b6e182140c9a98fa8b81f3f1f654b1c02f2f13be8246c65c20 +size 104308 diff --git a/assets/multilingual_results.png b/assets/multilingual_results.png new file mode 100644 index 0000000000000000000000000000000000000000..cb0fde26d8a1b34d059077942e8e44bf1728012c --- /dev/null +++ b/assets/multilingual_results.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d27c9bc9bdbeaebf6471eaefcfb5783ea91dd2adecce46093ac37edb1e070d9e +size 211708 diff --git a/assets/python-training-dynamics.png b/assets/python-training-dynamics.png new file mode 100644 index 0000000000000000000000000000000000000000..2450cbc3c4b128d6fd894a09c2787b01b17383a6 --- /dev/null +++ b/assets/python-training-dynamics.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a4a194b2bfa9d487a4b539ac28bdfc814a0a45140f9b718648fd01417053c45 +size 177644 diff --git a/assets/python_results.png b/assets/python_results.png new file mode 100644 index 0000000000000000000000000000000000000000..fc031034b8c749e6e1835504626917d8a843673e --- /dev/null +++ b/assets/python_results.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:341dd485ed0465024bb11f484104b1837aca16e39d8d4111977f5ceb32c01c32 +size 190583 diff --git a/assets/ultradata-code-l0-l3-overview.png b/assets/ultradata-code-l0-l3-overview.png new file mode 100644 index 0000000000000000000000000000000000000000..b1e2e42205874a06af114398bb6d98592fe4e228 --- /dev/null +++ b/assets/ultradata-code-l0-l3-overview.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01f4f23c1a8c3ad134eca73b7a3272b0ce389f3e2e489ceb55da4e748c8d3664 +size 3035924 diff --git a/assets/ultradata-code-l2.png b/assets/ultradata-code-l2.png new file mode 100644 index 0000000000000000000000000000000000000000..85bab48b506dae1fedcd479a5f15613c6df93aac --- /dev/null +++ b/assets/ultradata-code-l2.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2101a2e56f095ded9a3f1fd5b4f26949bf7e64560cb3fdacbf1776642c57736 +size 814507 diff --git a/classifiers/.DS_Store b/classifiers/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..e1689067a1a8406d1667411ea1cc5e2c1671d938 Binary files /dev/null and b/classifiers/.DS_Store differ diff --git a/classifiers/cpp/A_relevance.bin b/classifiers/cpp/A_relevance.bin new file mode 100644 index 0000000000000000000000000000000000000000..080ed8eb67f96de33426d61cd5c8dca55884f670 --- /dev/null +++ b/classifiers/cpp/A_relevance.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69d9fac735e3bcd6a5ac229971c10bba983d4ff054d8fcd555afaa8f820e4a79 +size 692293 diff --git a/classifiers/cpp/C_file_role.bin b/classifiers/cpp/C_file_role.bin new file mode 100644 index 0000000000000000000000000000000000000000..320e77f2f0f582527f82afd5446459dac769f877 --- /dev/null +++ b/classifiers/cpp/C_file_role.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26815d880b5880d63b1f70f55db0eae8b5ba87ed482f324f8967884e5c1e2f60 +size 3955829 diff --git a/classifiers/cpp/C_file_role.json b/classifiers/cpp/C_file_role.json new file mode 100644 index 0000000000000000000000000000000000000000..bd92e61738599e143a8a405c446b1ba534b6220b --- /dev/null +++ b/classifiers/cpp/C_file_role.json @@ -0,0 +1,11 @@ +{ + "idx2cat": { + "0": "ALGO", + "1": "CONFIG", + "2": "DATA", + "3": "EXCLUDE", + "4": "TEST", + "5": "TOOL", + "6": "WEB" + } +} diff --git a/classifiers/cpp/Q_quality.bin b/classifiers/cpp/Q_quality.bin new file mode 100644 index 0000000000000000000000000000000000000000..c03e07b9d429e0e1cfa9635cfa85c47fa6b611a3 --- /dev/null +++ b/classifiers/cpp/Q_quality.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd7b2072a5ec493cc7f87a7f214bdb0ab22f4db8872680ff9a8ce83d9f4afe23 +size 2645859 diff --git a/classifiers/cpp/Q_scaler.npz b/classifiers/cpp/Q_scaler.npz new file mode 100644 index 0000000000000000000000000000000000000000..6df4cca6d4c8e554ebe56e5ed7b759bde18ae450 --- /dev/null +++ b/classifiers/cpp/Q_scaler.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b55b8b39a3d254ce56e595f1c1f46d5f40f6f4e02c447934bb2620bb66b74e59 +size 8696 diff --git a/classifiers/csharp/A_relevance.bin b/classifiers/csharp/A_relevance.bin new file mode 100644 index 0000000000000000000000000000000000000000..5d087fad525f292a9419f47c8b8d8e7c5d80d186 --- /dev/null +++ b/classifiers/csharp/A_relevance.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8907abd9a500359a505984332517562b10bb99c486419ac46d38a8247485cd60 +size 692421 diff --git a/classifiers/csharp/C_file_role.bin b/classifiers/csharp/C_file_role.bin new file mode 100644 index 0000000000000000000000000000000000000000..84e5247e06bf1d2a6003eed670d0687479deb57c --- /dev/null +++ b/classifiers/csharp/C_file_role.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55e3a2c2f9066bb3c1027140a02af6f483a9f467f8d91b9fdd08ee8f9b9b2fb6 +size 5269621 diff --git a/classifiers/csharp/C_file_role.json b/classifiers/csharp/C_file_role.json new file mode 100644 index 0000000000000000000000000000000000000000..bd92e61738599e143a8a405c446b1ba534b6220b --- /dev/null +++ b/classifiers/csharp/C_file_role.json @@ -0,0 +1,11 @@ +{ + "idx2cat": { + "0": "ALGO", + "1": "CONFIG", + "2": "DATA", + "3": "EXCLUDE", + "4": "TEST", + "5": "TOOL", + "6": "WEB" + } +} diff --git a/classifiers/csharp/Q_quality.bin b/classifiers/csharp/Q_quality.bin new file mode 100644 index 0000000000000000000000000000000000000000..eae5a3f86235e52945a77cdc506bc443c7bf98f7 --- /dev/null +++ b/classifiers/csharp/Q_quality.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df3417dd4a7a05acf3ce330e1bb149dc28c7e397a72ac026002b344789064d12 +size 2645859 diff --git a/classifiers/csharp/Q_scaler.npz b/classifiers/csharp/Q_scaler.npz new file mode 100644 index 0000000000000000000000000000000000000000..101c1195302f296d252d1f7d0a40a892225c591d --- /dev/null +++ b/classifiers/csharp/Q_scaler.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab8d6cb4705304b14622df8fdf2aa770814305c6077408816c5afd0b5087139b +size 8696 diff --git a/classifiers/go/A_relevance.bin b/classifiers/go/A_relevance.bin new file mode 100644 index 0000000000000000000000000000000000000000..65f09e6fb437037f8b9563d30b17574c21bc4a96 --- /dev/null +++ b/classifiers/go/A_relevance.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7441a4dcbe6a75232053d30aa8987fe366cc55e1357e69b5cdfe51a1dade6e8c +size 692421 diff --git a/classifiers/go/C_file_role.bin b/classifiers/go/C_file_role.bin new file mode 100644 index 0000000000000000000000000000000000000000..ca9d699baf6081f88692cd5dcc2478e9d96accfa --- /dev/null +++ b/classifiers/go/C_file_role.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7993ac50b52a0fb2320fa1cf09571457d75e8d6c50379013dfa42e318da5684 +size 3955829 diff --git a/classifiers/go/C_file_role.json b/classifiers/go/C_file_role.json new file mode 100644 index 0000000000000000000000000000000000000000..bd92e61738599e143a8a405c446b1ba534b6220b --- /dev/null +++ b/classifiers/go/C_file_role.json @@ -0,0 +1,11 @@ +{ + "idx2cat": { + "0": "ALGO", + "1": "CONFIG", + "2": "DATA", + "3": "EXCLUDE", + "4": "TEST", + "5": "TOOL", + "6": "WEB" + } +} diff --git a/classifiers/go/Q_quality.bin b/classifiers/go/Q_quality.bin new file mode 100644 index 0000000000000000000000000000000000000000..5fcded15c2f19e34f329080967882a82fa836cbb --- /dev/null +++ b/classifiers/go/Q_quality.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab58f06e5e5099590dd0fcda2e902fbe3ed28e0383047a74b4fa84b4267e648e +size 2645859 diff --git a/classifiers/go/Q_scaler.npz b/classifiers/go/Q_scaler.npz new file mode 100644 index 0000000000000000000000000000000000000000..391ded4c86db73789d1db4a243a81957bebb30df --- /dev/null +++ b/classifiers/go/Q_scaler.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3204acf88ec63a714728f12ebe60d5f05463832c21bc1717ed9c9716ae5bd0cf +size 8696 diff --git a/classifiers/java/A_relevance.bin b/classifiers/java/A_relevance.bin new file mode 100644 index 0000000000000000000000000000000000000000..b21f4847046915bea20b9edd41c5a79955be468f --- /dev/null +++ b/classifiers/java/A_relevance.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3bb8f42061135312f693d252302bd7f6c9107f2a753d6e6c9cf885ecfbe60c3 +size 1446981 diff --git a/classifiers/java/C_file_role.bin b/classifiers/java/C_file_role.bin new file mode 100644 index 0000000000000000000000000000000000000000..7d2e2d19c0c765933828117b0db12892e59dd065 --- /dev/null +++ b/classifiers/java/C_file_role.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4699cf2c7b7db875d4edb8ba5147a118e8d10323f5512c92cf69e3045fc3fb07 +size 3955426 diff --git a/classifiers/java/C_file_role.json b/classifiers/java/C_file_role.json new file mode 100644 index 0000000000000000000000000000000000000000..bd92e61738599e143a8a405c446b1ba534b6220b --- /dev/null +++ b/classifiers/java/C_file_role.json @@ -0,0 +1,11 @@ +{ + "idx2cat": { + "0": "ALGO", + "1": "CONFIG", + "2": "DATA", + "3": "EXCLUDE", + "4": "TEST", + "5": "TOOL", + "6": "WEB" + } +} diff --git a/classifiers/java/Q_quality.bin b/classifiers/java/Q_quality.bin new file mode 100644 index 0000000000000000000000000000000000000000..b5ccda8506c22931a3886c4d70a807ddd9d45d33 --- /dev/null +++ b/classifiers/java/Q_quality.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:140ba3641b5db61b6e84d28e3d74237839f523e644d50e1966d6f1d09337a3b6 +size 2645859 diff --git a/classifiers/java/Q_scaler.npz b/classifiers/java/Q_scaler.npz new file mode 100644 index 0000000000000000000000000000000000000000..5b1ce008d243f4060a612530cde9733113d240e5 --- /dev/null +++ b/classifiers/java/Q_scaler.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:102ffd81a344ed33919c3c99996c48c9e2136e503ed3e406d47abc83c8ac1f44 +size 8696 diff --git a/classifiers/javascript/A_relevance.bin b/classifiers/javascript/A_relevance.bin new file mode 100644 index 0000000000000000000000000000000000000000..1c6d97da0e8cae7decd245d61ec9e728497301df --- /dev/null +++ b/classifiers/javascript/A_relevance.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb261d0ba07f624d0476ce5d27b315c1d71cdcf8c524797554885ff302af0ad5 +size 692421 diff --git a/classifiers/javascript/C_file_role.bin b/classifiers/javascript/C_file_role.bin new file mode 100644 index 0000000000000000000000000000000000000000..1cf3c79114f2e1ef9dfa679d39b5c06c67dd3235 --- /dev/null +++ b/classifiers/javascript/C_file_role.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:424a1934a321c683dedbd6613dd6021a0500a8248444ddfce135eb9d8bde6334 +size 5269621 diff --git a/classifiers/javascript/C_file_role.json b/classifiers/javascript/C_file_role.json new file mode 100644 index 0000000000000000000000000000000000000000..bd92e61738599e143a8a405c446b1ba534b6220b --- /dev/null +++ b/classifiers/javascript/C_file_role.json @@ -0,0 +1,11 @@ +{ + "idx2cat": { + "0": "ALGO", + "1": "CONFIG", + "2": "DATA", + "3": "EXCLUDE", + "4": "TEST", + "5": "TOOL", + "6": "WEB" + } +} diff --git a/classifiers/javascript/Q_quality.bin b/classifiers/javascript/Q_quality.bin new file mode 100644 index 0000000000000000000000000000000000000000..35a993101ffaf40f123c72b22e22021dee2d50cb --- /dev/null +++ b/classifiers/javascript/Q_quality.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96c06d31398978c87a36cd76b14fbecf7d2a503158b8d34c1f0e511f6f4c8e0d +size 4359523 diff --git a/classifiers/javascript/Q_scaler.npz b/classifiers/javascript/Q_scaler.npz new file mode 100644 index 0000000000000000000000000000000000000000..b7517f347385c923f72db245e774dbaa2b8b8369 --- /dev/null +++ b/classifiers/javascript/Q_scaler.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edb08e6d9131d5f137a548f54297745c9c6c579c0d5e78726e3ce3b9bf9c6693 +size 8696 diff --git a/classifiers/php/A_relevance.bin b/classifiers/php/A_relevance.bin new file mode 100644 index 0000000000000000000000000000000000000000..3de78e753c3042c003022e48c926628e41675e0e --- /dev/null +++ b/classifiers/php/A_relevance.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47a91f7423fd0ca3c3691bc83c32c790d14c99323335c81e95ae7ad7c60e3024 +size 692421 diff --git a/classifiers/php/C_file_role.bin b/classifiers/php/C_file_role.bin new file mode 100644 index 0000000000000000000000000000000000000000..03f63ad9f419a6355d9d305d6d67f36fd53c1a84 --- /dev/null +++ b/classifiers/php/C_file_role.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:067f21a3c1685407ef9c5c5bcfd08993c8fb1d34cf0f40a00af75eae46343821 +size 5269621 diff --git a/classifiers/php/C_file_role.json b/classifiers/php/C_file_role.json new file mode 100644 index 0000000000000000000000000000000000000000..bd92e61738599e143a8a405c446b1ba534b6220b --- /dev/null +++ b/classifiers/php/C_file_role.json @@ -0,0 +1,11 @@ +{ + "idx2cat": { + "0": "ALGO", + "1": "CONFIG", + "2": "DATA", + "3": "EXCLUDE", + "4": "TEST", + "5": "TOOL", + "6": "WEB" + } +} diff --git a/classifiers/php/Q_quality.bin b/classifiers/php/Q_quality.bin new file mode 100644 index 0000000000000000000000000000000000000000..8e7ce4ad42301e77de80b0333c05f136d418902a --- /dev/null +++ b/classifiers/php/Q_quality.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a14861f0b57adfc5746303dc7d6f7690c58b0a8f364ae744ff27204400c81d50 +size 2645859 diff --git a/classifiers/php/Q_scaler.npz b/classifiers/php/Q_scaler.npz new file mode 100644 index 0000000000000000000000000000000000000000..2f6b9a59ca0a78ba47fddacbd6c98c35bb897f5b --- /dev/null +++ b/classifiers/php/Q_scaler.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc285ab85d610fc7bee5bf48f6fefe53af94f92aaddebcaaeb2c1b36c4022d1f +size 8696 diff --git a/classifiers/python/A_relevance.bin b/classifiers/python/A_relevance.bin new file mode 100644 index 0000000000000000000000000000000000000000..ca6ac3ea531a28109fd2d3a7ae08743819c1df3f --- /dev/null +++ b/classifiers/python/A_relevance.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:215475344c065b4ec34bc92bced8b39b67c84b826bc95e95c407dc00a05cfab5 +size 691952 diff --git a/classifiers/python/C_file_role.bin b/classifiers/python/C_file_role.bin new file mode 100644 index 0000000000000000000000000000000000000000..01ca33bf0f837340f759b00d54b30e3d8e00d068 --- /dev/null +++ b/classifiers/python/C_file_role.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b681afb0528c975dba8cef9edb3e8a21db943debce42717fd4a35493b36284d +size 5269445 diff --git a/classifiers/python/C_file_role.json b/classifiers/python/C_file_role.json new file mode 100644 index 0000000000000000000000000000000000000000..bd92e61738599e143a8a405c446b1ba534b6220b --- /dev/null +++ b/classifiers/python/C_file_role.json @@ -0,0 +1,11 @@ +{ + "idx2cat": { + "0": "ALGO", + "1": "CONFIG", + "2": "DATA", + "3": "EXCLUDE", + "4": "TEST", + "5": "TOOL", + "6": "WEB" + } +} diff --git a/classifiers/python/Q_quality.bin b/classifiers/python/Q_quality.bin new file mode 100644 index 0000000000000000000000000000000000000000..a72f3b6b3169eac4f5229cf0ceb9b9dc1f1f4624 --- /dev/null +++ b/classifiers/python/Q_quality.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f10a4bb1f77644e43478e6b89cdf392b77a6c00d471de0bc80746ddbabae9165 +size 4359595 diff --git a/classifiers/python/Q_scaler.npz b/classifiers/python/Q_scaler.npz new file mode 100644 index 0000000000000000000000000000000000000000..d3d4da8317a71fb1f6e9b3334d8fd15f68c5877d --- /dev/null +++ b/classifiers/python/Q_scaler.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7afd8c690c96094a1f689c764345ec7ccb99d3b7d52d0fc2fce0f28f4ea273a +size 8696 diff --git a/classifiers/r/A_relevance.bin b/classifiers/r/A_relevance.bin new file mode 100644 index 0000000000000000000000000000000000000000..90fb4e14ac68abbf0176b12c84457a5777cb192c --- /dev/null +++ b/classifiers/r/A_relevance.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f72114badc2838ebd565f0fe3cb07d6ad5e4a4df6afa7f8d9f8e9aa023e5ccf +size 692421 diff --git a/classifiers/r/C_file_role.bin b/classifiers/r/C_file_role.bin new file mode 100644 index 0000000000000000000000000000000000000000..948c8fbcb0b97fc4b72b9713802a0fdf032d659f --- /dev/null +++ b/classifiers/r/C_file_role.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90fbac1d554770f77d0c939e9fa17d4508bef6334e9a5dce4e1280fcd40a7dea +size 2374773 diff --git a/classifiers/r/C_file_role.json b/classifiers/r/C_file_role.json new file mode 100644 index 0000000000000000000000000000000000000000..bd92e61738599e143a8a405c446b1ba534b6220b --- /dev/null +++ b/classifiers/r/C_file_role.json @@ -0,0 +1,11 @@ +{ + "idx2cat": { + "0": "ALGO", + "1": "CONFIG", + "2": "DATA", + "3": "EXCLUDE", + "4": "TEST", + "5": "TOOL", + "6": "WEB" + } +} diff --git a/classifiers/r/Q_quality.bin b/classifiers/r/Q_quality.bin new file mode 100644 index 0000000000000000000000000000000000000000..b7d75524e0092e328b5acb496c5cf59a6468530f --- /dev/null +++ b/classifiers/r/Q_quality.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2504cce34461f1bc2afd82a75c860d8aea49c196663ebaab706ae39367407cf1 +size 2645859 diff --git a/classifiers/r/Q_scaler.npz b/classifiers/r/Q_scaler.npz new file mode 100644 index 0000000000000000000000000000000000000000..79a4bd4ec6e40f1ca1b5ce2c549003d9715344e0 --- /dev/null +++ b/classifiers/r/Q_scaler.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bf48f51a045acdffd1363d5f1df2b05d8b7a8ae79e84644e6dc57f69a65a0c6 +size 8696 diff --git a/classifiers/ruby/A_relevance.bin b/classifiers/ruby/A_relevance.bin new file mode 100644 index 0000000000000000000000000000000000000000..348a6335fdde4c5e1afa8b3066e42dade1acf9bd --- /dev/null +++ b/classifiers/ruby/A_relevance.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a64077c5c4e611c6a131a34d9cec8f136eb2e5806176a70d5d47c9ada3dbdbe +size 692421 diff --git a/classifiers/ruby/C_file_role.bin b/classifiers/ruby/C_file_role.bin new file mode 100644 index 0000000000000000000000000000000000000000..db80f71bf8290ea1ac444bf3494dfb59a800ebc7 --- /dev/null +++ b/classifiers/ruby/C_file_role.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45dbadb3197a46da22e941df1d5b5964dfd4ed17f8ec4db268c4ded89a1b810b +size 2374773 diff --git a/classifiers/ruby/C_file_role.json b/classifiers/ruby/C_file_role.json new file mode 100644 index 0000000000000000000000000000000000000000..bd92e61738599e143a8a405c446b1ba534b6220b --- /dev/null +++ b/classifiers/ruby/C_file_role.json @@ -0,0 +1,11 @@ +{ + "idx2cat": { + "0": "ALGO", + "1": "CONFIG", + "2": "DATA", + "3": "EXCLUDE", + "4": "TEST", + "5": "TOOL", + "6": "WEB" + } +} diff --git a/classifiers/ruby/Q_quality.bin b/classifiers/ruby/Q_quality.bin new file mode 100644 index 0000000000000000000000000000000000000000..18764c11390c0117c5da24c7b244012f8871423b --- /dev/null +++ b/classifiers/ruby/Q_quality.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14fedf39c3ea5ec315a74c1238f0ecf1f0b7503bb64e14c1ee28cb15102f2af4 +size 4359523 diff --git a/classifiers/ruby/Q_scaler.npz b/classifiers/ruby/Q_scaler.npz new file mode 100644 index 0000000000000000000000000000000000000000..6d067b93ce7cbcaf25bd2a4c0f28c232a3f9b06c --- /dev/null +++ b/classifiers/ruby/Q_scaler.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27f4235353f970ab3a3eacdf6bd8664d54509dd8bcd1bfb77da3290af499bc2d +size 8696 diff --git a/classifiers/rust/A_relevance.bin b/classifiers/rust/A_relevance.bin new file mode 100644 index 0000000000000000000000000000000000000000..765c03046ebaff0d4c33de290c66d72034552e5e --- /dev/null +++ b/classifiers/rust/A_relevance.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eddfd7d6b83c0641415d51e21f52dfc77a40cb2be55c72bf27cfe88e3b5df012 +size 692421 diff --git a/classifiers/rust/C_file_role.bin b/classifiers/rust/C_file_role.bin new file mode 100644 index 0000000000000000000000000000000000000000..43514cfadce717e35745dabce22ad01293785e1e --- /dev/null +++ b/classifiers/rust/C_file_role.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82a07f0e33075139285f23a60737570488cddb6ff32030e0392e323480eb7ab3 +size 3955829 diff --git a/classifiers/rust/C_file_role.json b/classifiers/rust/C_file_role.json new file mode 100644 index 0000000000000000000000000000000000000000..bd92e61738599e143a8a405c446b1ba534b6220b --- /dev/null +++ b/classifiers/rust/C_file_role.json @@ -0,0 +1,11 @@ +{ + "idx2cat": { + "0": "ALGO", + "1": "CONFIG", + "2": "DATA", + "3": "EXCLUDE", + "4": "TEST", + "5": "TOOL", + "6": "WEB" + } +} diff --git a/classifiers/rust/Q_quality.bin b/classifiers/rust/Q_quality.bin new file mode 100644 index 0000000000000000000000000000000000000000..a2bbdc42dc98286b84c8c367485a917badb3ac55 --- /dev/null +++ b/classifiers/rust/Q_quality.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1cbbb287c15c6e15a20fa20875840a3722321d2160e4708dd61cc1ec27789400 +size 4359523 diff --git a/classifiers/rust/Q_scaler.npz b/classifiers/rust/Q_scaler.npz new file mode 100644 index 0000000000000000000000000000000000000000..3ab082f92b344839a7f7db06baabbfb6db65dae4 --- /dev/null +++ b/classifiers/rust/Q_scaler.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:158b621587980d32fef17aaa264ce26ec5013f3e93621a3078e7f0ead6532a41 +size 8696 diff --git a/classifiers/shell/A_relevance.bin b/classifiers/shell/A_relevance.bin new file mode 100644 index 0000000000000000000000000000000000000000..f6dfdda6abbeccef9b4e5d4cba10df7594a402b2 --- /dev/null +++ b/classifiers/shell/A_relevance.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:380ac6f65c1807bdb98c86269f5531aba2477d45c4d1071f0feff5ad5d20144c +size 692421 diff --git a/classifiers/shell/C_file_role.bin b/classifiers/shell/C_file_role.bin new file mode 100644 index 0000000000000000000000000000000000000000..313b2f9639caf8ffb57c062459b31f95e55de7eb --- /dev/null +++ b/classifiers/shell/C_file_role.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:afe48d99c1d277171ebfd0dae5d080bc4b711c6389d83de422642c61760ebea3 +size 3955829 diff --git a/classifiers/shell/C_file_role.json b/classifiers/shell/C_file_role.json new file mode 100644 index 0000000000000000000000000000000000000000..bd92e61738599e143a8a405c446b1ba534b6220b --- /dev/null +++ b/classifiers/shell/C_file_role.json @@ -0,0 +1,11 @@ +{ + "idx2cat": { + "0": "ALGO", + "1": "CONFIG", + "2": "DATA", + "3": "EXCLUDE", + "4": "TEST", + "5": "TOOL", + "6": "WEB" + } +} diff --git a/classifiers/shell/Q_quality.bin b/classifiers/shell/Q_quality.bin new file mode 100644 index 0000000000000000000000000000000000000000..5670fec1c12824c27df749732b25eae400ae1aa7 --- /dev/null +++ b/classifiers/shell/Q_quality.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2530296120c2359c79f2ca4b3bf215af0e036a7a452969d95bcbc9107a7aeaf +size 2645859 diff --git a/classifiers/shell/Q_scaler.npz b/classifiers/shell/Q_scaler.npz new file mode 100644 index 0000000000000000000000000000000000000000..e90a9d4848fbd78a0c100a46a8877374f5e6aef2 --- /dev/null +++ b/classifiers/shell/Q_scaler.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c71896eb7b30bcf414272cf8172fcba1763dc33657d4f7b7a1c22462e4417ae8 +size 8696 diff --git a/config.json b/config.json new file mode 100644 index 0000000000000000000000000000000000000000..526a7eb16ec6013caa7713435b41774956950e37 --- /dev/null +++ b/config.json @@ -0,0 +1,29 @@ +{ + "format": "code-selection-scorers/1", + "embedding_model": { + "repo_id": "Qwen/Qwen3-Embedding-0.6B", + "dim": 1024, + "training_max_tokens": 8192 + }, + "languages": [ + "cpp", + "csharp", + "go", + "java", + "javascript", + "php", + "python", + "r", + "ruby", + "rust", + "shell" + ], + "selection_policy": { + "algo_rel_min": 0.8, + "quality_min": 3.0, + "exclude_categories": [ + "EXCLUDE" + ], + "per_language": {} + } +} diff --git a/scripts/path_features.py b/scripts/path_features.py new file mode 100644 index 0000000000000000000000000000000000000000..81d3dbbbb8f26720311f64af697fcca778cd09eb --- /dev/null +++ b/scripts/path_features.py @@ -0,0 +1,143 @@ +"""Path featurization for the C / A / Q scorers. + +Only the algorithmic-relevance model A consumes the file path. It is turned +into a fixed-width vector by a hashing trick and concatenated to the code +embedding: + + features = [ embedding (1024) | path_hash (256) * weight (0.4) ] + +The algorithm is named ``crc32_signed_tokens_legacy`` in the checkpoints. +"legacy" is a name inherited from the training pipeline, not a deprecation -- +this is the only path featurizer, and every shipped scorer set uses it. + +Two details matter for reproducing training-time behaviour exactly: + +* Benchmark names (``leetcode``, ``humaneval``, ``mbpp``, ``codeforces`` ...) + are stripped from the path first, so the model cannot shortcut on them. +* Tokens are a bag of path components plus 3-character prefixes of the longer + ones. Position is discarded, which is what makes it tolerant of repository + layout differences. + +This is reproduced verbatim from the training-time implementation so a scorer +set is self-contained. +""" + +from __future__ import annotations + +import json +import re +import zlib +from typing import Sequence + +import numpy as np + +PATH_HASH_ALGORITHM = "crc32_signed_tokens_legacy" + + +# --------------------------------------------------------------------------- # +# read repository-relative paths out of a pyarrow table +# --------------------------------------------------------------------------- # +def _path_from_meta(value: object) -> str: + if isinstance(value, str): + try: + value = json.loads(value) + except (TypeError, ValueError): + return "" + if isinstance(value, dict): + path = value.get("file_path") + return path if isinstance(path, str) else "" + return "" + + +def extract_relative_paths(table, path_col: str = "relative_path") -> list[str]: + """Read paths from ``path_col``, falling back to ``meta.file_path``.""" + count = table.num_rows + if path_col in table.column_names: + paths = [ + "" if value is None else str(value) + for value in table.column(path_col).to_pylist() + ] + else: + paths = [""] * count + if "meta" in table.column_names and not all(paths): + paths = [ + path or _path_from_meta(meta) + for path, meta in zip(paths, table.column("meta").to_pylist()) + ] + return paths + + +# --------------------------------------------------------------------------- # +# benchmark-name stripping +# --------------------------------------------------------------------------- # +_SEPARATOR = r"[-_./\\\s]*" +_BENCHMARK_PATTERNS = ( + rf"human{_SEPARATOR}eval(?:{_SEPARATOR}plus|{_SEPARATOR}x)?", + rf"mbpp(?:{_SEPARATOR}plus)?", + r"multipl[-_./\\\s]+e", + rf"ds{_SEPARATOR}1000", + rf"crux{_SEPARATOR}eval", + rf"big{_SEPARATOR}code{_SEPARATOR}bench", + rf"live{_SEPARATOR}code{_SEPARATOR}bench", + rf"code{_SEPARATOR}contests?", + r"leetcode", + r"codeforces", + r"atcoder", + r"acm", +) +_BENCHMARK_RE = re.compile( + rf"(? str: + """Remove benchmark names so the model cannot shortcut on them.""" + path = str(relative_path or "").replace("\\", "/").lower() + return _BENCHMARK_RE.sub("/", path) + + +# --------------------------------------------------------------------------- # +# hashing trick +# --------------------------------------------------------------------------- # +def path_tokens(relative_path: str) -> list[str]: + """Lowercased path components plus 3-char prefixes of longer tokens.""" + if not relative_path: + return [] + raw = _TOKEN_SPLIT.split(sanitize_benchmark_path(relative_path)) + tokens = [t for t in raw if t] + extra = [t[:3] for t in tokens if len(t) > 3] + return tokens + extra + + +def path_hash_vector(relative_path: str, dim: int) -> np.ndarray: + """crc32 hashing-trick bag-of-tokens, L2-normalized, shape [dim].""" + vec = np.zeros(dim, dtype=np.float32) + for tok in path_tokens(relative_path): + encoded = tok.encode("utf-8") + bucket = zlib.crc32(encoded) % dim + sign = 1.0 if (zlib.crc32(b"s:" + encoded) & 1) == 0 else -1.0 + vec[bucket] += sign + norm = float(np.linalg.norm(vec)) + if norm > 0.0: + vec /= norm + return vec + + +def build_feature_matrix( + embeddings: Sequence[Sequence[float]], + paths: Sequence[str], + *, + path_hash_dim: int, + path_feature_weight: float, +) -> np.ndarray: + """Build ``[ embedding | path_hash * weight ]``, shape [N, D+H].""" + rows = [] + for embedding, path in zip(embeddings, paths): + emb = np.asarray(embedding, dtype=np.float32).reshape(-1) + hashed = path_hash_vector(path, path_hash_dim) * float(path_feature_weight) + rows.append(np.concatenate([emb, hashed.astype(np.float32)])) + if not rows: + return np.zeros((0, 0), dtype=np.float32) + return np.stack(rows).astype(np.float32) diff --git a/scripts/score_parquet.py b/scripts/score_parquet.py new file mode 100644 index 0000000000000000000000000000000000000000..f5213a7c18a2ef72cd2c5feab22ecaafdea3df15 --- /dev/null +++ b/scripts/score_parquet.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +"""Score parquet shards with one language's C / A / Q scorers. + +One row is one source file; a parquet shard holds many rows. Four columns are +appended to every input table and the result is written under ``--output-root``, +mirroring the input directory structure: + + category argmax of the file-role classifier C + cls_confidence softmax max probability of C + algo_rel_score sigmoid output of the algorithmic-relevance model A + quality_score fused 0..10 output of the quality model Q + +With ``--apply-policy`` a boolean ``selected`` column is added as well, using +the retention rule ``algo_rel_score >= t AND category not in excluded AND +quality_score >= q``. It marks rows rather than dropping them. + +Inputs must carry an ``embedding`` column. A ``relative_path`` column (or a +``meta`` struct with ``file_path``) is required only when the scorers +consume path features — the script checks and refuses to run silently without +them. + +This is a single-process scorer. It walks a directory recursively, but has no +worker pool and no resume ledger, so for corpus-scale runs drive it from your own +scheduler (one invocation per shard or per subtree) rather than pointing it at +millions of rows in one go. +""" + +from __future__ import annotations + +import argparse +import json +import sys +import time +from pathlib import Path + +import numpy as np +import pyarrow as pa +import pyarrow.parquet as pq + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from path_features import extract_relative_paths # noqa: E402 +from scorers import LanguageScorers # noqa: E402 + + +def collect_inputs(input_path: Path) -> list[Path]: + if input_path.is_file(): + return [input_path] + files = sorted(input_path.rglob("*.parquet")) + if not files: + raise FileNotFoundError(f"no parquet files under {input_path}") + return files + + +def score_table(table, scorers: LanguageScorers, batch_size: int): + embeddings = table.column("embedding").to_pylist() + if scorers.needs_paths(): + paths = extract_relative_paths(table, scorers.path_column) + if not any(paths): + raise ValueError( + "these scorers use path features but no 'relative_path' column " + "(or meta.file_path) was found in the input" + ) + else: + paths = [""] * len(embeddings) + + columns = {"category": [], "cls_confidence": [], "algo_rel_score": [], "quality_score": []} + for start in range(0, len(embeddings), batch_size): + stop = start + batch_size + chunk = np.asarray(embeddings[start:stop], dtype=np.float32) + scored = scorers.score(chunk, paths[start:stop]) + columns["category"].extend(scored["category"]) + for key in ("cls_confidence", "algo_rel_score", "quality_score"): + columns[key].extend(np.asarray(scored[key]).reshape(-1).tolist()) + return columns + + +def load_policy_defaults(language: str) -> dict: + """Read the retention policy defaults for one language from config.json. + + Kept out of the argparse defaults on purpose: the cuts are a policy choice, + not a property of the models, so they live in one editable place rather than + baked into this file. ``scorers.py`` never reads this — only the CLI does. + + ``selection_policy`` holds the cuts that apply to every language. An optional + ``per_language`` block overrides any of them for a single language, so each + language can carry its own operating point without touching the others:: + + "selection_policy": { + "algo_rel_min": 0.8, + "per_language": {"r": {"algo_rel_min": 0.9}} + } + + Precedence is CLI flag > per_language > shared default. + """ + path = Path(__file__).resolve().parent.parent / "config.json" + try: + policy = json.loads(path.read_text())["selection_policy"] + override = (policy.get("per_language") or {}).get(language, {}) + merged = {**policy, **override} + return { + "algo_rel_min": float(merged["algo_rel_min"]), + "quality_min": float(merged["quality_min"]), + "exclude_categories": ",".join(merged["exclude_categories"]), + "source": "per_language" if override else "shared", + } + except (OSError, KeyError, ValueError, TypeError) as exc: + raise SystemExit( + f"could not read selection_policy from {path}: {exc}. " + "Pass --algo-rel-min / --quality-min / --exclude-categories explicitly, " + "or restore the config file." + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--scorers", required=True, help="scorer-set dir for one language, e.g. classifiers/cpp") + parser.add_argument("--input", help="parquet file or directory of parquet files") + parser.add_argument("--output-root", help="destination directory for scored parquet") + parser.add_argument("--batch-size", type=int, default=1024) + parser.add_argument("--device", default="cpu") + parser.add_argument("--limit", type=int, default=0, help="score at most N input parquet files (0 = all)") + parser.add_argument("--describe", action="store_true", help="print the scorer-set configuration and exit") + parser.add_argument("--apply-policy", action="store_true", help="also write a boolean 'selected' column") + parser.add_argument("--algo-rel-min", type=float, help="default: config.json selection_policy") + parser.add_argument("--quality-min", type=float, help="default: config.json selection_policy") + parser.add_argument("--exclude-categories", help="comma separated categories to veto; default: config.json") + args = parser.parse_args() + + scorers = LanguageScorers.load(args.scorers, device=args.device) + + if args.describe: + print(json.dumps(scorers.describe(), indent=2, ensure_ascii=False)) + return + + if not args.input or not args.output_root: + parser.error("--input and --output-root are required unless --describe is given") + + input_path = Path(args.input) + output_root = Path(args.output_root) + files = collect_inputs(input_path) + if args.limit: + files = files[: args.limit] + root = input_path if input_path.is_dir() else input_path.parent + + # Resolve the retention cuts only when they are actually needed, so a plain + # scoring run does not fail on a missing/edited config.json. + algo_rel_min = quality_min = None + excluded = set() + if args.apply_policy: + defaults = load_policy_defaults(scorers.language) + algo_rel_min = ( + args.algo_rel_min if args.algo_rel_min is not None + else defaults["algo_rel_min"] + ) + quality_min = ( + args.quality_min if args.quality_min is not None + else defaults["quality_min"] + ) + categories = ( + args.exclude_categories if args.exclude_categories is not None + else defaults["exclude_categories"] + ) + excluded = {c.strip() for c in categories.split(",") if c.strip()} + print( + f"policy [{scorers.language}, {defaults['source']} defaults]: " + f"algo_rel_score >= {algo_rel_min} AND category not in " + f"{sorted(excluded)} AND quality_score >= {quality_min}", + flush=True, + ) + + total_rows = 0 + for index, in_path in enumerate(files, 1): + started = time.time() + table = pq.read_table(in_path) + columns = score_table(table, scorers, args.batch_size) + + out = table + for name, values, arrow_type in ( + ("category", columns["category"], pa.string()), + ("cls_confidence", columns["cls_confidence"], pa.float32()), + ("algo_rel_score", columns["algo_rel_score"], pa.float32()), + ("quality_score", columns["quality_score"], pa.float32()), + ): + if name in out.column_names: + out = out.drop([name]) + out = out.append_column(name, pa.array(values, type=arrow_type)) + + if args.apply_policy: + selected = [ + bool( + relevance >= algo_rel_min + and category not in excluded + and quality >= quality_min + ) + for relevance, category, quality in zip( + columns["algo_rel_score"], columns["category"], columns["quality_score"] + ) + ] + if "selected" in out.column_names: + out = out.drop(["selected"]) + out = out.append_column("selected", pa.array(selected, type=pa.bool_())) + + out_path = output_root / in_path.relative_to(root) + out_path.parent.mkdir(parents=True, exist_ok=True) + pq.write_table(out, out_path) + total_rows += table.num_rows + print( + f"[{index}/{len(files)}] {in_path.name}: {table.num_rows} rows " + f"in {time.time() - started:.1f}s -> {out_path}", + flush=True, + ) + + print(f"done: {len(files)} parquet files, {total_rows} rows -> {output_root}") + + +if __name__ == "__main__": + main() diff --git a/scripts/scorers.py b/scripts/scorers.py new file mode 100644 index 0000000000000000000000000000000000000000..037efc8ffbf4a179d2ed91445c1b2406b40fb682 --- /dev/null +++ b/scripts/scorers.py @@ -0,0 +1,324 @@ +"""Loaders and inference for the C / A / Q code-selection scorers. + +A scorer set is a flat directory, one per language, such as ``classifiers/cpp``:: + + / + C_file_role.bin C_file_role.json # classifier + its class names + A_relevance.bin # checkpoint carries its own config + Q_quality.bin # architecture read from its shapes + Q_scaler.npz # Q's StandardScaler (mean / scale) + +All five files are required. Layer widths are recovered from the checkpoints' +own tensor shapes rather than declared in config, so the only metadata shipped +alongside the weights is ``C_file_role.json`` — the index-to-category map, the +one thing the weights cannot carry. + +``LanguageScorers.load(dir)`` reads the artifacts and configures itself from +what they declare; nothing about the featurization is hard-coded per language. + +All three models consume a precomputed code embedding. Embeddings must come +from the same encoder and dimension used at training time — the loader checks +the dimension, but cannot detect a wrong encoder at the right dimension. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import numpy as np +import torch +import torch.nn as nn + +from path_features import PATH_HASH_ALGORITHM, build_feature_matrix + + +# --------------------------------------------------------------------------- # +# architectures (must match the training scripts' saved state_dicts) +# --------------------------------------------------------------------------- # +class FileRoleClassifier(nn.Module): + """C: Linear/LayerNorm/GELU/Dropout stack -> num_classes logits.""" + + def __init__(self, input_dim, hidden_dims, num_classes, dropout=0.3): + super().__init__() + layers = [] + prev = input_dim + for i, h in enumerate(hidden_dims): + layers += [ + nn.Linear(prev, h), + nn.LayerNorm(h), + nn.GELU(), + nn.Dropout(dropout if i == 0 else dropout * 0.7), + ] + prev = h + layers.append(nn.Linear(prev, num_classes)) + self.net = nn.Sequential(*layers) + + def forward(self, x): + return self.net(x) + + +class RelevanceMLP(nn.Module): + """A: two hidden ReLU layers -> single logit.""" + + def __init__(self, in_dim, hidden1, hidden2): + super().__init__() + self.fc1 = nn.Linear(in_dim, hidden1) + self.fc2 = nn.Linear(hidden1, hidden2) + self.fc3 = nn.Linear(hidden2, 1) + + def forward(self, x): + x = torch.relu(self.fc1(x)) + x = torch.relu(self.fc2(x)) + return self.fc3(x).squeeze(-1) + + +class _QualityBackbone(nn.Module): + """Q's shared trunk. + + Kept as its own module holding a ``self.net`` Sequential because the saved + state_dicts are keyed ``backbone.net.0.*``. Inlining the Sequential into + QualityMLP would renumber the keys to ``backbone.0.*`` and fail to load. + """ + + def __init__(self, in_dim, hidden_dim, dropout): + super().__init__() + mid = max(64, hidden_dim // 2) + self.net = nn.Sequential( + nn.Linear(in_dim, hidden_dim), + nn.GELU(), + nn.LayerNorm(hidden_dim), + nn.Dropout(dropout), + nn.Linear(hidden_dim, mid), + nn.GELU(), + nn.LayerNorm(mid), + nn.Dropout(dropout), + ) + + def forward(self, x): + return self.net(x) + + +class QualityMLP(nn.Module): + """Q: shared trunk -> zero-score head + 1..10 ordinal head. + + Final score is ``(1 - sigmoid(zero)) * (1 + sum(sigmoid(ordinal)))``: the + zero head gates out config/data/no-logic files, the ordinal head places + everything else on 1..10. + """ + + def __init__(self, in_dim, hidden_dim=512, dropout=0.1, ordinal_num_classes=10): + super().__init__() + mid = max(64, hidden_dim // 2) + self.backbone = _QualityBackbone(in_dim, hidden_dim, dropout) + self.zero_head = nn.Linear(mid, 1) + self.ordinal_levels = ordinal_num_classes - 1 + self.ordinal_head = nn.Linear(mid, self.ordinal_levels) + + def forward(self, x): + features = self.backbone(x) + return self.zero_head(features).squeeze(-1), self.ordinal_head(features) + + +# --------------------------------------------------------------------------- # +# standard scaler without an sklearn version dependency +# --------------------------------------------------------------------------- # +class _Scaler: + """(x - mean) / scale, loaded from ``Q_scaler.npz``.""" + + def __init__(self, mean: np.ndarray, scale: np.ndarray, source: str): + self.mean = mean.astype(np.float32) + self.scale = scale.astype(np.float32) + self.n_features_in = int(self.mean.size) + self.source = source + + @classmethod + def load(cls, scorer_dir: Path) -> "_Scaler": + npz = scorer_dir / "Q_scaler.npz" + payload = np.load(npz) + return cls(payload["mean"], payload["scale"], npz.name) + + def transform(self, x: np.ndarray) -> np.ndarray: + return ((x - self.mean) / self.scale).astype(np.float32) + + +# --------------------------------------------------------------------------- # +# scorer set +# --------------------------------------------------------------------------- # +class LanguageScorers: + """Loads one language's scorer set and scores batches of embeddings.""" + + def __init__(self, scorer_dir, device="cpu"): + self.scorer_dir = Path(scorer_dir) + self.device = torch.device(device) + self.language = self.scorer_dir.name + + # -- loading ---------------------------------------------------------- # + @classmethod + def load(cls, scorer_dir, device="cpu") -> "LanguageScorers": + self = cls(scorer_dir, device) + self._load_file_role() + self._load_relevance() + self._load_quality() + if self.quality_in_dim != self.embedding_dim: + raise ValueError( + f"Q expects {self.quality_in_dim}-d input but C implies " + f"{self.embedding_dim}-d embeddings" + ) + return self + + def _load_file_role(self) -> None: + cfg = json.loads((self.scorer_dir / "C_file_role.json").read_text()) + + state = torch.load( + self.scorer_dir / "C_file_role.bin", map_location=self.device + ) + # Layer widths come from the checkpoint itself: every Linear weight is + # [out, in], so the stack of 2-D weights gives input_dim, the hidden + # widths and num_classes. Dropout is inactive under eval(). + linears = [v.shape for k, v in state.items() + if k.endswith("weight") and v.ndim == 2] + input_dim = int(linears[0][1]) + hidden_dims = [int(s[0]) for s in linears[:-1]] + num_classes = int(linears[-1][0]) + + model = FileRoleClassifier(input_dim, hidden_dims, num_classes) + model.load_state_dict(state) + self.file_role_model = model.eval().to(self.device) + self.embedding_dim = input_dim + + # The one thing weights cannot carry: which class each index means. + self.idx2cat = {int(k): v for k, v in cfg["idx2cat"].items()} + if len(self.idx2cat) != num_classes: + raise ValueError( + f"C_file_role.json maps {len(self.idx2cat)} categories but the " + f"checkpoint has {num_classes} output units" + ) + + def _load_relevance(self) -> None: + # Named explicitly rather than globbed: in a flat scorer-set directory a + # glob for "*.bin" would also match the C and Q checkpoints. + ckpt = torch.load( + self.scorer_dir / "A_relevance.bin", map_location=self.device + ) + arch = ckpt["arch"] + model = RelevanceMLP(arch["in_dim"], arch["hidden1"], arch["hidden2"]) + model.load_state_dict(ckpt["state_dict"]) + self.relevance_model = model.eval().to(self.device) + self.relevance_in_dim = int(arch["in_dim"]) + + cfg = ckpt["feature_config"] + if not cfg.get("use_path_feature", False): + raise ValueError( + "this A checkpoint declares no path features; every scorer set " + "in this release feeds A a 1024-d embedding plus a 256-d path hash" + ) + self.path_hash_dim = int(cfg["path_hash_dim"]) + self.path_feature_weight = float(cfg["path_feature_weight"]) + self.path_column = "relative_path" + self.path_algorithm = PATH_HASH_ALGORITHM + + expected = self.embedding_dim + self.path_hash_dim + if expected != self.relevance_in_dim: + raise ValueError( + f"A expects {self.relevance_in_dim}-d input but the configured " + f"featurization produces {expected}-d" + ) + + def _load_quality(self) -> None: + self.quality_scaler = _Scaler.load(self.scorer_dir) + state = torch.load(self.scorer_dir / "Q_quality.bin", map_location=self.device) + + # Shapes carry the architecture: the trunk's first Linear is + # [hidden_dim, in_dim] and the ordinal head is [num_classes - 1, mid]. + # No config file is needed, and load_state_dict below is the real guard + # — a differently-headed checkpoint fails on key names, not on a flag. + in_dim = self.quality_scaler.n_features_in or int( + state["backbone.net.0.weight"].shape[1] + ) + hidden_dim = int(state["backbone.net.0.weight"].shape[0]) + ordinal_num_classes = int(state["ordinal_head.weight"].shape[0]) + 1 + + model = QualityMLP(in_dim, hidden_dim, ordinal_num_classes=ordinal_num_classes) + model.load_state_dict(state) + self.quality_model = model.eval().to(self.device) + self.quality_in_dim = in_dim + + # -- inference -------------------------------------------------------- # + def predict_file_role(self, embeddings: np.ndarray): + with torch.no_grad(): + logits = self.file_role_model(torch.from_numpy(embeddings).to(self.device)) + probs = torch.softmax(logits, dim=1) + confidence, index = probs.max(dim=1) + roles = [self.idx2cat.get(int(i), "EXCLUDE") for i in index.cpu().numpy()] + return roles, confidence.cpu().numpy() + + def predict_relevance(self, embeddings: np.ndarray, paths) -> np.ndarray: + features = build_feature_matrix( + embeddings, + paths, + path_hash_dim=self.path_hash_dim, + path_feature_weight=self.path_feature_weight, + ) + with torch.no_grad(): + tensor = torch.from_numpy(np.ascontiguousarray(features, dtype=np.float32)) + logits = self.relevance_model(tensor.to(self.device)) + scores = torch.sigmoid(logits).cpu().numpy() + return np.asarray(scores).reshape(-1) + + def predict_quality(self, embeddings: np.ndarray) -> np.ndarray: + scaled = self.quality_scaler.transform(embeddings) + with torch.no_grad(): + zero_logit, ordinal_logits = self.quality_model( + torch.from_numpy(scaled).to(self.device) + ) + p_zero = torch.sigmoid(zero_logit) + expected_positive = 1.0 + torch.sigmoid(ordinal_logits).sum(dim=1) + out = (1.0 - p_zero) * expected_positive + return np.asarray(out.cpu().numpy()).reshape(-1) + + def score(self, embeddings, paths=None) -> dict: + """Score a batch. Returns the four columns as numpy arrays / lists.""" + embeddings = np.ascontiguousarray(embeddings, dtype=np.float32) + if embeddings.ndim != 2 or embeddings.shape[1] != self.embedding_dim: + raise ValueError( + f"expected embeddings of shape [N, {self.embedding_dim}], " + f"got {tuple(embeddings.shape)}" + ) + if paths is None: + paths = [""] * len(embeddings) + roles, confidence = self.predict_file_role(embeddings) + return { + "category": roles, + "cls_confidence": confidence, + "algo_rel_score": self.predict_relevance(embeddings, paths), + "quality_score": self.predict_quality(embeddings), + } + + def needs_paths(self) -> bool: + """True when any model consumes the file path. Always true: A does.""" + return self.path_hash_dim > 0 + + def describe(self) -> dict: + return { + "language": self.language, + "scorer_dir": str(self.scorer_dir), + "embedding_dim": self.embedding_dim, + "C": { + "input_dim": self.embedding_dim, + "categories": [self.idx2cat[i] for i in sorted(self.idx2cat)], + }, + "A": { + "input_dim": self.relevance_in_dim, + "path_hash_dim": self.path_hash_dim, + "path_feature_weight": self.path_feature_weight, + "path_algorithm": self.path_algorithm, + "path_column": self.path_column, + }, + "Q": { + "strategy": "two_stage", + "input_dim": self.quality_in_dim, + "ordinal_num_classes": self.quality_model.ordinal_levels + 1, + "scaler_source": self.quality_scaler.source, + }, + }