| import os |
| import subprocess |
| from pathlib import Path |
| from src.safe_subprocess import run |
| import sys |
| import re |
|
|
| ENABLE_SYNTAX_CHECK = False |
|
|
| def eval_script(path: Path): |
| result = run(["rdmd", "-unittest", str(path)], timeout_seconds=15) |
| if "might not be correctly installed" in result.stderr: |
| raise Exception("D is not correctly installed") |
| |
| if result.timeout: |
| status = "Timeout" |
| elif result.exit_code == 0: |
| status = "OK" |
| elif "Error:" in result.stderr: |
| status = "SyntaxError" |
| else: |
| status = "Exception" |
| |
| return { |
| "status": status, |
| "exit_code": result.exit_code, |
| "stdout": result.stdout, |
| "stderr": result.stderr, |
| } |
|
|
| DIR = "d-keep-code_davinci_001_temp_0.2" |
| def main(): |
| directory = Path(Path(__file__).parent, "..", "datasets", DIR).resolve() |
|
|
| count = {"OK": 0, "Timeout": 0, "Exception": 0, "SyntaxError": 0} |
| for filename in os.listdir(directory): |
| path = Path.joinpath(directory, filename) |
| r = eval_script(path) |
| status = r["status"] |
| count[status] += 1 |
|
|
| if ENABLE_SYNTAX_CHECK and status == "SyntaxError": |
| error_msgs = r["stderr"].split("\n") |
| with open(path) as source_file: |
| lines = source_file.readlines() |
| unittest_line_start = lines.index("unittest\n") |
| unittest_line_end = len(lines) |
| for err_msg_line in error_msgs: |
| matched_parts = re.match(r"(\/?.*?\.[\w:]+\/.*.d)\(([0-9]+)\): Error: (.*)", err_msg_line[2:-1]) |
| _file, line_num = matched_parts[1], int(matched_parts[2]) |
| if unittest_line_start <= line_num and line_num <= unittest_line_end: |
| print("===============") |
| print(path, "contains error in unit test part") |
| print(error_msgs) |
| print("===============") |
|
|
| filename = filename.split(".")[0] |
| print(f"Dlang,{filename},{status}") |
|
|
| print(DIR + ":" + str(count)) |
|
|
| if __name__ == "__main__": |
| main() |
|
|
