Spaces:
Running
Running
Create app.py
Browse files
app.py
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import gradio as gr
|
| 2 |
+
import duckdb
|
| 3 |
+
import urllib.parse
|
| 4 |
+
|
| 5 |
+
PARQUET_PATH = "v3_repos.parquet"
|
| 6 |
+
|
| 7 |
+
con = duckdb.connect()
|
| 8 |
+
|
| 9 |
+
text = """\
|
| 10 |
+

|
| 11 |
+
# Am I in The Stack?
|
| 12 |
+
|
| 13 |
+
[The Stack v3](https://huggingface.co/datasets/HuggingFaceCode/stack-v3-train) is \
|
| 14 |
+
a 15.9 TB dataset of source code across 713 programming languages from 173M repositories, \
|
| 15 |
+
crawled from GitHub in 2025.
|
| 16 |
+
|
| 17 |
+
We want to give developers agency over their source code \
|
| 18 |
+
by letting them decide whether or not it should be used to develop and evaluate \
|
| 19 |
+
machine learning models.
|
| 20 |
+
|
| 21 |
+
Enter your GitHub username below to check if any of your repositories are in The Stack v3. \
|
| 22 |
+
If your code is found, you will get a pre-filled opt-out link \
|
| 23 |
+
that submits a properly formatted removal request on your behalf. \
|
| 24 |
+
**Please use this tool to submit opt-out requests** rather than opening issues manually β \
|
| 25 |
+
it ensures your request can be processed quickly and correctly.
|
| 26 |
+
"""
|
| 27 |
+
|
| 28 |
+
opt_out_text_template = """\
|
| 29 |
+
### Opt-out
|
| 30 |
+
|
| 31 |
+
If you want your data to be removed from The Stack and model training, \
|
| 32 |
+
open an issue with <a href="https://github.com/bigcode-project/opt-out-v2/issues/new?title={title}&body={body}" target="_blank">this link</a> \
|
| 33 |
+
(if the link doesn't work, try right-clicking and opening it in a new tab) \
|
| 34 |
+
or visit [https://github.com/bigcode-project/opt-out-v2/issues/new?&template=opt-out-request.md](https://github.com/bigcode-project/opt-out-v2/issues/new?&template=opt-out-request.md).\
|
| 35 |
+
"""
|
| 36 |
+
|
| 37 |
+
opt_out_issue_title = "Opt-out request for {username}"
|
| 38 |
+
opt_out_issue_body = """\
|
| 39 |
+
I request that the following data is removed from The Stack:
|
| 40 |
+
|
| 41 |
+
{repo_list}
|
| 42 |
+
|
| 43 |
+
_Note_: If you don't want all repositories removed, edit the list above. \
|
| 44 |
+
To exclude everything under your username, replace the list with a single "all" entry.
|
| 45 |
+
"""
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def issue_url(username, repos):
|
| 49 |
+
title = urllib.parse.quote(opt_out_issue_title.format(username=username))
|
| 50 |
+
body = urllib.parse.quote(
|
| 51 |
+
opt_out_issue_body.format(repo_list=" - " + "\n - ".join(repos))
|
| 52 |
+
)
|
| 53 |
+
return opt_out_text_template.format(title=title, body=body)
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def check_username(username):
|
| 57 |
+
username = username.strip()
|
| 58 |
+
if not username:
|
| 59 |
+
return "", ""
|
| 60 |
+
|
| 61 |
+
repos = con.execute(
|
| 62 |
+
f"""SELECT repo FROM read_parquet('{PARQUET_PATH}')
|
| 63 |
+
WHERE "user" = $1 ORDER BY repo""",
|
| 64 |
+
[username.lower()],
|
| 65 |
+
).fetchall()
|
| 66 |
+
repos = [row[0] for row in repos]
|
| 67 |
+
|
| 68 |
+
if not repos:
|
| 69 |
+
return "**No**, your code is not in The Stack v3.", ""
|
| 70 |
+
|
| 71 |
+
repo_word = "repository" if len(repos) == 1 else "repositories"
|
| 72 |
+
lines = [f"**Yes**, there is code from **{len(repos)} {repo_word}** in The Stack v3:\n"]
|
| 73 |
+
for repo in repos:
|
| 74 |
+
full = f"{username}/{repo}"
|
| 75 |
+
lines.append(f"[{full}](https://github.com/{full})\n")
|
| 76 |
+
output_md = "\n".join(lines).strip()
|
| 77 |
+
|
| 78 |
+
full_names = [f"{username}/{r}" for r in repos]
|
| 79 |
+
return output_md, issue_url(username, full_names)
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
with gr.Blocks() as demo:
|
| 83 |
+
with gr.Row():
|
| 84 |
+
_, col, _ = gr.Column(scale=1), gr.Column(scale=6), gr.Column(scale=1)
|
| 85 |
+
with col:
|
| 86 |
+
gr.Markdown(text)
|
| 87 |
+
username = gr.Text("", label="Your GitHub username:")
|
| 88 |
+
check_button = gr.Button("Check!")
|
| 89 |
+
repos_md = gr.Markdown()
|
| 90 |
+
opt_out_md = gr.Markdown()
|
| 91 |
+
check_button.click(check_username, [username], [repos_md, opt_out_md])
|
| 92 |
+
|
| 93 |
+
demo.launch()
|