File size: 5,382 Bytes
3738348 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 | """
Download a multi-language code corpus from GitHub for base pretraining.
Downloads tarballs of several popular repositories across Python, JS/TS,
Rust, Go, C/C++, and Java. Extracts source files and concatenates them
into a single training corpus.
"""
import io
import os
import tarfile
import time
import requests
DATA_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "data")
RAW_DIR = os.path.join(DATA_DIR, "raw")
CORPUS_PATH = os.path.join(DATA_DIR, "corpus.txt")
# Repos to download (user/repo, branch, max_bytes_per_repo)
REPOS = [
("python/cpython", "main", 8_000_000), # Python, C
("microsoft/TypeScript", "main", 8_000_000), # TypeScript
("rust-lang/rust", "master", 8_000_000), # Rust
("golang/go", "master", 8_000_000), # Go
("nginx/nginx", "master", 5_000_000), # C
("json-iterator/go", "master", 3_000_000), # Go
("pytorch/pytorch", "main", 8_000_000), # C++, Python, CUDA
("tokio-rs/tokio", "master", 5_000_000), # Rust
("numpy/numpy", "main", 5_000_000), # Python, C
("axios/axios", "v1.x", 3_000_000), # JavaScript
("lodash/lodash", "master", 3_000_000), # JavaScript
("redis/redis", "unstable", 5_000_000), # C
]
# File extensions to include
CODE_EXTENSIONS = {
".py", ".js", ".ts", ".mjs", ".jsx", ".tsx",
".rs", ".go", ".c", ".h", ".cpp", ".cc", ".cxx",
".hpp", ".hxx", ".cu", ".cuh", ".java",
".sh", ".bash", ".yml", ".yaml", ".toml",
".cfg", ".ini", ".json", ".xml", ".sql",
".md", ".txt", ".rst",
}
# Directories to skip
SKIP_DIRS = {
"test", "tests", "testing", "__pycache__", ".git",
"node_modules", "vendor", "third_party", "thirdparty",
"dist", "build", "target", ".github", "docs",
"benchmark", "benchmarks", "examples", "example",
"fixtures", "testdata", "test_data", "mocks",
}
def download_repo_tarball(user_repo: str, branch: str) -> bytes | None:
"""Download a repository tarball from GitHub."""
url = f"https://codeload.github.com/{user_repo}/tar.gz/refs/heads/{branch}"
print(f" Downloading {user_repo}@{branch}...")
try:
r = requests.get(url, timeout=120, stream=True)
if r.status_code != 200:
# Try tags endpoint
url2 = f"https://codeload.github.com/{user_repo}/tar.gz/refs/tags/{branch}"
r = requests.get(url2, timeout=120, stream=True)
if r.status_code != 200:
print(f" FAILED (status {r.status_code})")
return None
content = r.content
print(f" Downloaded {len(content) / 1e6:.1f} MB")
return content
except Exception as e:
print(f" ERROR: {e}")
return None
def extract_code_files(tarball_bytes: bytes, max_bytes: int) -> list[str]:
"""Extract code files from a tarball, up to max_bytes total."""
files = []
total = 0
try:
with tarfile.open(fileobj=io.BytesIO(tarball_bytes), mode="r:gz") as tar:
for member in tar.getmembers():
if not member.isfile():
continue
path = member.name
parts = path.split("/")
# Skip unwanted directories
if any(skip in parts for skip in SKIP_DIRS):
continue
# Check extension
_, ext = os.path.splitext(path)
if ext.lower() not in CODE_EXTENSIONS:
continue
# Skip very large files
if member.size > 500_000:
continue
try:
f = tar.extractfile(member)
if f is None:
continue
content = f.read()
try:
text = content.decode("utf-8", errors="ignore")
except Exception:
continue
files.append(text)
total += len(text)
if total >= max_bytes:
break
except Exception:
continue
except Exception as e:
print(f" Extraction error: {e}")
print(f" Extracted {len(files)} files, {total / 1e6:.1f} MB of code")
return files
def main():
os.makedirs(RAW_DIR, exist_ok=True)
os.makedirs(DATA_DIR, exist_ok=True)
all_code = []
for user_repo, branch, max_bytes in REPOS:
tarball = download_repo_tarball(user_repo, branch)
if tarball is None:
continue
files = extract_code_files(tarball, max_bytes)
all_code.extend(files)
time.sleep(1) # Be polite
# Write corpus
print(f"\nTotal files: {len(all_code)}")
total_size = sum(len(f) for f in all_code)
print(f"Total corpus size: {total_size / 1e6:.1f} MB")
with open(CORPUS_PATH, "w", encoding="utf-8") as f:
for i, code in enumerate(all_code):
# Add document separator
f.write(code)
f.write("\n<|endoftext|>\n")
print(f"Corpus written to {CORPUS_PATH}")
return total_size
if __name__ == "__main__":
main()
|