Upload test_battery.py with huggingface_hub

test_battery.py

@@ -0,0 +1,516 @@
+#!/usr/bin/env python3
+"""
+Harbour Test Battery - Generates code, compiles with harbour, evaluates with qwen3.6:35b
+"""
+
+import json
+import time
+import subprocess
+import requests
+import tempfile
+import os
+from pathlib import Path
+from datetime import datetime
+
+OLLAMA_URL = "http://localhost:11434/api/generate"
+MODEL = "qwen3.6:35b"
+HARBOUR = "/home/fivetech/harbour/bin/linux/gcc/harbour"
+WORK_DIR = Path("/home/fivetech/finetune/test_output")
+WORK_DIR.mkdir(exist_ok=True)
+
+def query_ollama(prompt, system="", timeout=300):
+    payload = {
+        "model": MODEL,
+        "prompt": prompt,
+        "stream": False,
+        "options": {"temperature": 0.2, "num_predict": 3000, "top_p": 0.9}
+    }
+    if system:
+        payload["system"] = system
+    try:
+        start = time.time()
+        r = requests.post(OLLAMA_URL, json=payload, timeout=timeout)
+        elapsed = time.time() - start
+        data = r.json()
+        return {
+            "response": data.get("response", ""),
+            "eval_count": data.get("eval_count", 0),
+            "duration": elapsed,
+            "tps": data.get("eval_count", 0) / max(data.get("eval_duration", 1) / 1e9, 0.001),
+            "error": None
+        }
+    except Exception as e:
+        return {"response": "", "error": str(e), "eval_count": 0, "duration": 0, "tps": 0}
+
+def compile_harbour(code):
+    """Compile code with harbour, return (success, error_msg, obj_exists)"""
+    prg_file = WORK_DIR / "test.prg"
+    prg_file.write_text(code)
+    
+    try:
+        result = subprocess.run(
+            [HARBOUR, str(prg_file), "-n", "-w"],
+            capture_output=True, text=True, timeout=30
+        )
+        obj_file = WORK_DIR / "test.obj"
+        success = result.returncode == 0
+        obj_exists = obj_file.exists()
+        error = result.stderr.strip() if result.stderr else ""
+        if not success and not error:
+            error = result.stdout.strip()
+        return success, error, obj_exists
+    except subprocess.TimeoutExpired:
+        return False, "Compilation timeout", False
+    except Exception as e:
+        return False, str(e), False
+
+def clean_code(response):
+    """Extract code from model response, remove markdown."""
+    lines = response.split('\n')
+    in_code = False
+    code_lines = []
+    skip_explanation = True
+    
+    for line in lines:
+        stripped = line.strip()
+        
+        # Skip markdown
+        if stripped.startswith('```'):
+            in_code = not in_code
+            continue
+        
+        if in_code:
+            code_lines.append(line)
+            skip_explanation = False
+        elif skip_explanation:
+            # Detect start of code
+            upper = stripped.upper()
+            if any(upper.startswith(kw) for kw in [
+                'FUNCTION', 'PROCEDURE', 'LOCAL', 'STATIC', 'PUBLIC',
+                'PRIVATE', 'MEMVAR', '#DEFINE', '#INCLUDE', 'CLASS',
+                'METHOD', 'RETURN', 'SET', 'REQUEST'
+            ]):
+                in_code = True
+                code_lines.append(line)
+                skip_explanation = False
+    
+    if not code_lines:
+        # Fallback: take everything
+        code_lines = response.split('\n')
+    
+    return '\n'.join(code_lines).strip()
+
+# ============================================================
+# TEST DEFINITIONS - Based on dataset patterns
+# ============================================================
+
+TESTS = [
+    # ---- BASIC SYNTAX ----
+    {
+        "id": "SYNTAX_01", "category": "Basic Syntax", "name": "Variable types and declarations",
+        "prompt": "Write a Harbour program that declares LOCAL variables of each type (numeric, character, logical, date, nil), prints them with ValType(), and uses proper Hungarian notation.",
+        "expected_keywords": ["LOCAL", "ValType", "FUNCTION"],
+        "min_lines": 8,
+    },
+    {
+        "id": "SYNTAX_02", "category": "Basic Syntax", "name": "Preprocessor defines",
+        "prompt": "Write Harbour preprocessor definitions for application constants: app name, version, max records, date format. Use #define and show conditional compilation with #ifdef.",
+        "expected_keywords": ["#DEFINE", "#IFDEF", "#ENDIF"],
+        "min_lines": 6,
+    },
+    {
+        "id": "SYNTAX_03", "category": "Basic Syntax", "name": "String operations",
+        "prompt": "Write a Harbour function that takes a full name string and returns initials. Use AllTrim, Upper, Left, At, SubStr, and Space functions.",
+        "expected_keywords": ["FUNCTION", "AllTrim", "Upper", "Left", "At", "SubStr"],
+        "min_lines": 6,
+    },
+    {
+        "id": "SYNTAX_04", "category": "Basic Syntax", "name": "Date functions",
+        "prompt": "Write a Harbour function that calculates the number of business days between two dates, excluding weekends. Use Date(), DOW(), and date arithmetic.",
+        "expected_keywords": ["FUNCTION", "Date", "DOW"],
+        "min_lines": 8,
+    },
+    {
+        "id": "SYNTAX_05", "category": "Basic Syntax", "name": "Type conversion",
+        "prompt": "Write Harbour code that converts between all types: Str, Val, CTOD, DTOC, ASC, Chr, Transform. Show edge cases.",
+        "expected_keywords": ["Str", "Val", "CTOD", "DTOC"],
+        "min_lines": 8,
+    },
+
+    # ---- CONTROL FLOW ----
+    {
+        "id": "CTRL_01", "category": "Control Flow", "name": "IF/ELSEIF/ENDIF",
+        "prompt": "Write a Harbour function that classifies employee salary into tax brackets using IF/ELSEIF/ELSE/ENDIF. Include 5 brackets and error handling.",
+        "expected_keywords": ["FUNCTION", "IF", "ELSEIF", "ELSE", "ENDIF"],
+        "min_lines": 10,
+    },
+    {
+        "id": "CTRL_02", "category": "Control Flow", "name": "DO CASE",
+        "prompt": "Write a Harbour function using DO CASE to convert month number (1-12) to season name. Handle invalid input with OTHERWISE.",
+        "expected_keywords": ["DO CASE", "CASE", "OTHERWISE", "ENDCASE"],
+        "min_lines": 8,
+    },
+    {
+        "id": "CTRL_03", "category": "Control Flow", "name": "FOR/NEXT loop",
+        "prompt": "Write a Harbour function using FOR/NEXT to calculate the sum of all prime numbers below 100. Include STEP and EXIT.",
+        "expected_keywords": ["FOR", "TO", "NEXT", "IF", "EXIT"],
+        "min_lines": 10,
+    },
+    {
+        "id": "CTRL_04", "category": "Control Flow", "name": "DO WHILE",
+        "prompt": "Write a Harbour function using DO WHILE to implement the Euclidean algorithm for GCD. Include LOOP and EXIT.",
+        "expected_keywords": ["DO WHILE", "ENDDO", "IF", "LOOP", "EXIT"],
+        "min_lines": 6,
+    },
+    {
+        "id": "CTRL_05", "category": "Control Flow", "name": "SCAN/ENDSCAN",
+        "prompt": "Write Harbour code using SCAN/ENDSCAN to find the longest string in an array. Include NEXT clause.",
+        "expected_keywords": ["SCAN", "ENDSCAN"],
+        "min_lines": 6,
+    },
+    {
+        "id": "CTRL_06", "category": "Control Flow", "name": "FOR EACH",
+        "prompt": "Write Harbour code using FOR EACH to count word frequencies in a string. Use a hash for storage.",
+        "expected_keywords": ["FOR EACH", "NEXT", ":="],
+        "min_lines": 8,
+    },
+
+    # ---- FUNCTIONS ----
+    {
+        "id": "FUNC_01", "category": "Functions", "name": "Parameters and return",
+        "prompt": "Write a Harbour function with default parameters, pass-by-reference using @, and return an array. Include proper Hungarian notation.",
+        "expected_keywords": ["FUNCTION", "LOCAL", "RETURN"],
+        "min_lines": 6,
+    },
+    {
+        "id": "FUNC_02", "category": "Functions", "name": "Recursion",
+        "prompt": "Write a recursive Harbour function for Fibonacci numbers with memoization using a hash. Include base case and error handling.",
+        "expected_keywords": ["FUNCTION", "IF", "RETURN"],
+        "min_lines": 8,
+    },
+    {
+        "id": "FUNC_03", "category": "Functions", "name": "Variable scope",
+        "prompt": "Write Harbour code demonstrating LOCAL, STATIC, PRIVATE, PUBLIC variables. Show scope differences with nested function calls.",
+        "expected_keywords": ["LOCAL", "STATIC", "PRIVATE", "PUBLIC"],
+        "min_lines": 8,
+    },
+    {
+        "id": "FUNC_04", "category": "Functions", "name": "Code blocks",
+        "prompt": "Write Harbour code using code blocks: AEval with {|x| x*2}, AScan, ASort with custom sort. Show evaluation with Eval().",
+        "expected_keywords": ["AEval", "AScan", "ASort", "Eval"],
+        "min_lines": 6,
+    },
+    {
+        "id": "FUNC_05", "category": "Functions", "name": "Error handling",
+        "prompt": "Write a Harbour function with BEGIN SEQUENCE/RECOVER/END SEQUENCE for file reading. Include DEFAULT and BREAK.",
+        "expected_keywords": ["BEGIN SEQUENCE", "RECOVER", "END SEQUENCE"],
+        "min_lines": 8,
+    },
+
+    # ---- ARRAYS ----
+    {
+        "id": "ARRAY_01", "category": "Arrays", "name": "Array operations",
+        "prompt": "Write Harbour functions for: create 2D array, AAdd elements, ASort with custom order, AScan by value, ASize to resize. Include error handling.",
+        "expected_keywords": ["ARRAY", "AAdd", "ASort", "AScan", "ASize"],
+        "min_lines": 8,
+    },
+    {
+        "id": "ARRAY_02", "category": "Arrays", "name": "Hash operations",
+        "prompt": "Write Harbour code using hashes: create, add keys, iterate with FOR EACH, merge two hashes, check key existence with HB_HHasKey, convert to array.",
+        "expected_keywords": [":=", "FOR EACH", "HB_HHasKey"],
+        "min_lines": 8,
+    },
+    {
+        "id": "ARRAY_03", "category": "Arrays", "name": "Sorting algorithm",
+        "prompt": "Implement QuickSort in Harbour for an array of numbers. Include partition logic and proper recursion.",
+        "expected_keywords": ["FUNCTION", "LOCAL", "IF", "RETURN"],
+        "min_lines": 12,
+    },
+
+    # ---- OOP ----
+    {
+        "id": "OOP_01", "category": "OOP", "name": "Class definition",
+        "prompt": "Write a Harbour class Person with DATA (name, age), METHOD (New constructor, GetName, SetAge), and CLASSDATA. Include validation in SetAge.",
+        "expected_keywords": ["CLASS", "DATA", "METHOD", "RETURN"],
+        "min_lines": 10,
+    },
+    {
+        "id": "OOP_02", "category": "OOP", "name": "Inheritance",
+        "prompt": "Write Harbour classes: Shape (base), Circle (derived) with area() method. Show inheritance syntax and method override.",
+        "expected_keywords": ["CLASS", "METHOD", "INHERIT"],
+        "min_lines": 10,
+    },
+    {
+        "id": "OOP_03", "category": "OOP", "name": "Operator overloading",
+        "prompt": "Write a Harbour class Vec2 for 2D vectors. Overload + and - operators. Include magnitude and normalize methods.",
+        "expected_keywords": ["CLASS", "METHOD", "OPERATOR"],
+        "min_lines": 12,
+    },
+    {
+        "id": "OOP_04", "category": "OOP", "name": "Singleton pattern",
+        "prompt": "Implement Singleton pattern in Harbour for a config manager. Ensure only one instance exists.",
+        "expected_keywords": ["CLASS", "CLASSDATA", "METHOD"],
+        "min_lines": 10,
+    },
+
+    # ---- DATABASE ----
+    {
+        "id": "DB_01", "category": "Database", "name": "Basic RDD",
+        "prompt": "Write Harbour code that creates a DBF file, opens it, appends records, and closes properly. Use DBCreate and DBUseArea.",
+        "expected_keywords": ["DBCreate", "DBUseArea", "DBAppend", "DBCLOSEALL"],
+        "min_lines": 10,
+    },
+    {
+        "id": "DB_02", "category": "Database", "name": "Indexing",
+        "prompt": "Write Harbour code creating an index on a DBF field using RDD. Include ORDSCOPE for range queries.",
+        "expected_keywords": ["ORDCREATE", "ORDSCOPE"],
+        "min_lines": 8,
+    },
+    {
+        "id": "DB_03", "category": "Database", "name": "DBEval",
+        "prompt": "Write Harbour code using DBEval to process all records: count, sum field values, and mark records meeting a condition.",
+        "expected_keywords": ["DBEval", "FOR", "WHILE"],
+        "min_lines": 8,
+    },
+
+    # ---- FILE I/O ----
+    {
+        "id": "FILE_01", "category": "File I/O", "name": "Text file read/write",
+        "prompt": "Write Harbour functions to read a text file line by line and write processed output. Use FCreate, FOpen, FRead, FWrite, FClose, FEof.",
+        "expected_keywords": ["FCreate", "FOpen", "FRead", "FWrite", "FClose", "FEof"],
+        "min_lines": 10,
+    },
+    {
+        "id": "FILE_02", "category": "File I/O", "name": "Directory listing",
+        "prompt": "Write Harbour code using Directory() to list files with a pattern, get file size and date, and process each file.",
+        "expected_keywords": ["Directory", "LEN", "FOR"],
+        "min_lines": 6,
+    },
+
+    # ---- COMPLEX ----
+    {
+        "id": "CMPX_01", "category": "Complex", "name": "CSV parser",
+        "prompt": "Write a Harbour CSV parser that reads a CSV file, handles quoted fields, and returns an array of arrays. Include error handling.",
+        "expected_keywords": ["FUNCTION", "LOCAL", "FClose", "FEof"],
+        "min_lines": 15,
+    },
+    {
+        "id": "CMPX_02", "category": "Complex", "name": "INI file reader",
+        "prompt": "Write a Harbour INI file parser. Read sections, keys, and values into a hash. Handle comments and empty lines.",
+        "expected_keywords": ["FUNCTION", "LOCAL", "HASH"],
+        "min_lines": 12,
+    },
+    {
+        "id": "CMPX_03", "category": "Complex", "name": "String template engine",
+        "prompt": "Write a Harbour template engine replacing {{variable}} placeholders with hash values. Include error handling for missing keys.",
+        "expected_keywords": ["FUNCTION", "LOCAL", "STRTRAN"],
+        "min_lines": 8,
+    },
+    {
+        "id": "CMPX_04", "category": "Complex", "name": "Logger",
+        "prompt": "Write a Harbour logging system with DEBUG/INFO/WARN/ERROR levels, timestamp, file output, and configurable level filtering.",
+        "expected_keywords": ["FUNCTION", "LOCAL", "FClose"],
+        "min_lines": 12,
+    },
+    {
+        "id": "CMPX_05", "category": "Complex", "name": "Base64 encoder",
+        "prompt": "Write a Harbour Base64 encoder/decode function. Use Asc(), Chr(), and bit operations.",
+        "expected_keywords": ["FUNCTION", "LOCAL", "Asc", "Chr"],
+        "min_lines": 10,
+    },
+    {
+        "id": "CMPX_06", "category": "Complex", "name": "JSON serializer",
+        "prompt": "Write a Harbour function that serializes a hash to JSON string. Handle strings, numbers, booleans, arrays, and nested objects.",
+        "expected_keywords": ["FUNCTION", "LOCAL", "HB_IsHash"],
+        "min_lines": 15,
+    },
+    {
+        "id": "CMPX_07", "category": "Complex", "name": "LRU Cache",
+        "prompt": "Write a Harbour LRU cache class with get/set/delete, TTL expiration, and max size. Use a hash and an array for ordering.",
+        "expected_keywords": ["CLASS", "DATA", "METHOD"],
+        "min_lines": 15,
+    },
+    {
+        "id": "CMPX_08", "category": "Complex", "name": "SQL-like query on arrays",
+        "prompt": "Write a Harbour function that filters an array of hashes like SQL WHERE clause. Support =, <>, >, <, LIKE operators.",
+        "expected_keywords": ["FUNCTION", "LOCAL", "FOR"],
+        "min_lines": 12,
+    },
+    {
+        "id": "CMPX_09", "category": "Complex", "name": "Rate limiter",
+        "prompt": "Write a Harbour rate limiter class: max N requests per M seconds. Use timestamps and a queue.",
+        "expected_keywords": ["CLASS", "METHOD", "LOCAL"],
+        "min_lines": 12,
+    },
+    {
+        "id": "CMPX_10", "category": "Complex", "name": "Config file writer",
+        "prompt": "Write a Harbour config manager that saves/loads settings to JSON file. Include defaults, validation, and typed getters.",
+        "expected_keywords": ["FUNCTION", "LOCAL", "FClose"],
+        "min_lines": 12,
+    },
+
+    # ---- BUGGY CODE TO FIX ----
+    {
+        "id": "FIX_01", "category": "Bug Fix", "name": "Null pointer",
+        "prompt": "Fix this Harbour code that crashes when array is empty:\nLOCAL a := {}\n? a[1]",
+        "expected_keywords": ["IF", "LEN", "RETURN"],
+        "min_lines": 3,
+    },
+    {
+        "id": "FIX_02", "category": "Bug Fix", "name": "Wrong loop bounds",
+        "prompt": "Fix this code that skips last element:\nLOCAL a := {10,20,30}\nFOR i := 1 TO LEN(a)-1\n   ? a[i]\nNEXT",
+        "expected_keywords": ["FOR", "TO", "LEN"],
+        "min_lines": 3,
+    },
+    {
+        "id": "FIX_03", "category": "Bug Fix", "name": "String concat error",
+        "prompt": "Fix this code that fails on nil values:\nLOCAL cName := NIL\n? 'Hello ' + cName",
+        "expected_keywords": ["IF", "LOCAL", "RETURN"],
+        "min_lines": 3,
+    },
+
+    # ---- HARBOUR-SPECIFIC ----
+    {
+        "id": "HARB_01", "category": "Harbour-Specific", "name": "HB_* functions",
+        "prompt": "Write Harbour code using HB_IsString, HB_IsNumeric, HB_IsArray, HB_IsHash, HB_IsNil to validate function arguments. Include proper error messages.",
+        "expected_keywords": ["HB_IsString", "HB_IsNumeric", "IF"],
+        "min_lines": 6,
+    },
+    {
+        "id": "HARB_02", "category": "Harbour-Specific", "name": "Regex",
+        "prompt": "Write a Harbour function using HB_RegEx to validate email addresses. Use HB_RegExCompile and HB_RegExMatch.",
+        "expected_keywords": ["HB_RegEx", "FUNCTION"],
+        "min_lines": 6,
+    },
+    {
+        "id": "HARB_03", "category": "Harbour-Specific", "name": "Serialization",
+        "prompt": "Write Harbour code that serializes a hash to binary with HB_Serialize and deserializes with HB_Deserialize.",
+        "expected_keywords": ["HB_Serialize", "HB_Deserialize"],
+        "min_lines": 6,
+    },
+    {
+        "id": "HARB_04", "category": "Harbour-Specific", "name": "File path operations",
+        "prompt": "Write Harbour code using hb_DirBuild, hb_DirNameGet, hb_FileNameGet, hb_PathNormalize for cross-platform file handling.",
+        "expected_keywords": ["hb_Dir", "hb_File", "hb_Path"],
+        "min_lines": 6,
+    },
+]
+
+# ============================================================
+# MAIN
+# ============================================================
+
+def main():
+    print("=" * 70)
+    print("HARBOUR CODE GENERATION TEST BATTERY")
+    print(f"Model: {MODEL}")
+    print(f"Tests: {len(TESTS)}")
+    print(f"Harbour: {HARBOUR}")
+    print(f"Started: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
+    print("=" * 70)
+    
+    SYSTEM = """You are an expert Harbour programmer. Write clean, correct, COMPILABLE Harbour code.
+Use Hungarian notation: n=numeric, c=character, l=logical, a=array, o=object, d=date.
+Use 3-space indentation.
+Do NOT include explanations or markdown. Only raw Harbour code.
+End functions with RETURN and END FUNCTION."""
+
+    results = []
+    compile_pass = 0
+    compile_fail = 0
+    
+    for i, test in enumerate(TESTS, 1):
+        print(f"\n[{i:2d}/{len(TESTS)}] {test['id']}: {test['name']}")
+        
+        # Query model
+        result = query_ollama(test["prompt"], SYSTEM)
+        
+        if result["error"]:
+            print(f"  MODEL ERROR: {result['error']}")
+            results.append({"test": test, "model_error": result["error"], "compile": False, "compile_error": ""})
+            continue
+        
+        # Clean response
+        code = clean_code(result["response"])
+        
+        # Check for expected keywords
+        keywords_found = [kw for kw in test["expected_keywords"] if kw.upper() in code.upper()]
+        keywords_missing = [kw for kw in test["expected_keywords"] if kw.upper() not in code.upper()]
+        
+        # Compile
+        success, error, obj = compile_harbour(code)
+        
+        status = "PASS" if success else "FAIL"
+        if success:
+            compile_pass += 1
+        else:
+            compile_fail += 1
+        
+        print(f"  Compile: {status} | Keywords: {len(keywords_found)}/{len(test['expected_keywords'])} | TPS: {result['tps']:.0f}")
+        if keywords_missing:
+            print(f"  Missing keywords: {', '.join(keywords_missing)}")
+        if not success and error:
+            # Show first error only
+            first_error = error.split('\n')[0][:120]
+            print(f"  Error: {first_error}")
+        
+        results.append({
+            "test": test,
+            "code": code[:3000],
+            "compile_success": success,
+            "compile_error": error[:500] if error else "",
+            "keywords_found": keywords_found,
+            "keywords_missing": keywords_missing,
+            "tokens": result["eval_count"],
+            "tps": result["tps"],
+            "duration": result["duration"],
+            "lines": code.count('\n') + 1,
+        })
+    
+    # Summary by category
+    print("\n" + "=" * 70)
+    print("RESULTS SUMMARY")
+    print("=" * 70)
+    
+    categories = {}
+    for r in results:
+        cat = r["test"]["category"]
+        if cat not in categories:
+            categories[cat] = {"pass": 0, "fail": 0, "total": 0}
+        categories[cat]["total"] += 1
+        if r.get("compile_success"):
+            categories[cat]["pass"] += 1
+        else:
+            categories[cat]["fail"] += 1
+    
+    print(f"\n{'Category':<20} {'Pass':<6} {'Fail':<6} {'Rate':<8}")
+    print("-" * 45)
+    for cat, data in sorted(categories.items()):
+        rate = data["pass"] / data["total"] * 100 if data["total"] > 0 else 0
+        print(f"{cat:<20} {data['pass']:<6} {data['fail']:<6} {rate:.0f}%")
+    
+    print(f"\n{'TOTAL':<20} {compile_pass:<6} {compile_fail:<6} {compile_pass/len(results)*100:.0f}%")
+    print(f"Total tests: {len(results)}")
+    
+    total_tokens = sum(r.get("tokens", 0) for r in results)
+    total_time = sum(r.get("duration", 0) for r in results)
+    print(f"Total tokens: {total_tokens:,}")
+    print(f"Total time: {total_time:.1f}s")
+    
+    # Save
+    output = Path("/home/fivetech/finetune/test_baseline_qwen36.json")
+    with open(output, "w") as f:
+        json.dump({
+            "model": MODEL,
+            "timestamp": datetime.now().isoformat(),
+            "compile_pass": compile_pass,
+            "compile_fail": compile_fail,
+            "compile_rate": compile_pass / len(results) * 100,
+            "categories": categories,
+            "results": results,
+        }, f, indent=2, ensure_ascii=False)
+    
+    print(f"\nResults saved to: {output}")
+
+if __name__ == "__main__":
+    main()