Hugging Face's logo

Henry65
/
RepoSim4Py

Feature Extraction
Transformers
PyTorch
roberta
code-understanding
unixcoder
text-embeddings-inference
Model card Files
xet
 Community
1
Henry65 commited on
Commit
e42969a
·
1 Parent(s): d148ccd

Update RepoPipeline.py

Browse files
Files changed (1) hide show
  1. RepoPipeline.py +3 -3
RepoPipeline.py CHANGED
@@ -113,7 +113,7 @@ def extract_information(repos, headers=None):
113
  with tarfile.open(fileobj=response.raw, mode="r|gz") as tar:
114
  for member in tar:
115
  # 2. Extracting codes and docs.
116
- if (member.name.endswith(".py") and member.isfile()) is True:
117
  try:
118
  file_content = tar.extractfile(member).read().decode("utf-8")
119
  # extract_code_and_docs
@@ -127,7 +127,7 @@ def extract_information(repos, headers=None):
127
  except SyntaxError as e:
128
  tqdm.write(f"[-] SyntaxError in {member.name}, skipping: \n{e}")
129
  # 3. Extracting readme.
130
- elif ((member.name == "README.md" or member.name == "README.rst") and member.isfile()) is True:
131
  try:
132
  file_content = tar.extractfile(member).read().decode("utf-8")
133
  # extract readme
@@ -141,7 +141,7 @@ def extract_information(repos, headers=None):
141
  except SyntaxError as e:
142
  tqdm.write(f"[-] SyntaxError in {member.name}, skipping: \n{e}")
143
  # 4. Extracting requirements.
144
- elif (member.name == "requirements.txt" and member.isfile()) is True:
145
  try:
146
  lines = tar.extractfile(member).readlines().decode("utf-8")
147
  # extract readme
 
113
  with tarfile.open(fileobj=response.raw, mode="r|gz") as tar:
114
  for member in tar:
115
  # 2. Extracting codes and docs.
116
+ if member.name.endswith(".py") and member.isfile():
117
  try:
118
  file_content = tar.extractfile(member).read().decode("utf-8")
119
  # extract_code_and_docs
 
127
  except SyntaxError as e:
128
  tqdm.write(f"[-] SyntaxError in {member.name}, skipping: \n{e}")
129
  # 3. Extracting readme.
130
+ elif (member.name == "README.md" or member.name == "README.rst") and member.isfile():
131
  try:
132
  file_content = tar.extractfile(member).read().decode("utf-8")
133
  # extract readme
 
141
  except SyntaxError as e:
142
  tqdm.write(f"[-] SyntaxError in {member.name}, skipping: \n{e}")
143
  # 4. Extracting requirements.
144
+ elif member.name == "requirements.txt" and member.isfile():
145
  try:
146
  lines = tar.extractfile(member).readlines().decode("utf-8")
147
  # extract readme