Upload Slayer GPT tokenizer model archive
Browse files
docs/assets/tokenizer_bpe_pipeline.png
CHANGED
|
|
docs/tokenizer_bpe_pipeline.tex
CHANGED
|
@@ -2,7 +2,7 @@
|
|
| 2 |
\usepackage[T1]{fontenc}
|
| 3 |
\usepackage[utf8]{inputenc}
|
| 4 |
\usepackage{tikz}
|
| 5 |
-
\usetikzlibrary{arrows.meta,positioning,shapes.geometric}
|
| 6 |
|
| 7 |
\definecolor{slayerRed}{RGB}{190,42,38}
|
| 8 |
\definecolor{ink}{RGB}{28,30,34}
|
|
@@ -64,5 +64,10 @@
|
|
| 64 |
\node[font=\sffamily\bfseries\large, text=ink, above=1.0cm of pairs] {
|
| 65 |
Jak powstal tokenizer Slayer GPT: byte-level BPE, nie morfologia
|
| 66 |
};
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 67 |
\end{tikzpicture}
|
| 68 |
\end{document}
|
|
|
|
| 2 |
\usepackage[T1]{fontenc}
|
| 3 |
\usepackage[utf8]{inputenc}
|
| 4 |
\usepackage{tikz}
|
| 5 |
+
\usetikzlibrary{arrows.meta,backgrounds,positioning,shapes.geometric}
|
| 6 |
|
| 7 |
\definecolor{slayerRed}{RGB}{190,42,38}
|
| 8 |
\definecolor{ink}{RGB}{28,30,34}
|
|
|
|
| 64 |
\node[font=\sffamily\bfseries\large, text=ink, above=1.0cm of pairs] {
|
| 65 |
Jak powstal tokenizer Slayer GPT: byte-level BPE, nie morfologia
|
| 66 |
};
|
| 67 |
+
|
| 68 |
+
\begin{scope}[on background layer]
|
| 69 |
+
\fill[white] ([xshift=-0.35cm,yshift=-0.35cm]current bounding box.south west)
|
| 70 |
+
rectangle ([xshift=0.35cm,yshift=0.35cm]current bounding box.north east);
|
| 71 |
+
\end{scope}
|
| 72 |
\end{tikzpicture}
|
| 73 |
\end{document}
|