import re
import math
import json
import string
import tempfile
import os
from collections import Counter
from html.parser import HTMLParser
from typing import Dict, List, Tuple, Any, Optional
import xml.etree.ElementTree as ET
import gradio as gr
# =====================================================================
# CUSTOM CORE PARSERS & HELPERS (NO EXTERNAL NLP LIBRARIES)
# =====================================================================
class SimpleHTMLInspector(HTMLParser):
"""
A lightweight, pure-Python HTML inspector utilizing standard HTMLParser.
Fulfills educational HTML analysis without external BeautifulSoup or library dependencies.
"""
def __init__(self):
super().__init__()
self.tags: List[str] = []
self.content_map: List[Tuple[str, str]] = []
self.tag_stack: List[str] = []
def handle_starttag(self, tag: str, attrs: List[Tuple[str, Optional[str]]]):
self.tags.append(tag)
self.tag_stack.append(tag)
def handle_data(self, data: str):
cleaned = data.strip()
if cleaned:
current_tag = self.tag_stack[-1] if self.tag_stack else "text"
# Ignore style/script data for clean text preview
if current_tag not in ["script", "style"]:
self.content_map.append((current_tag, cleaned))
def handle_endtag(self, tag: str):
if self.tag_stack and self.tag_stack[-1] == tag:
self.tag_stack.pop()
def parse_csv_custom(text: str, delimiter: str = ",") -> Tuple[List[str], List[List[str]], int, int]:
"""
A custom state-machine CSV parser built from scratch.
Complies with rules: No import csv, manages quote boundaries, escaped quotes, and newlines correctly.
"""
lines = text.splitlines()
if not lines:
return [], [], 0, 0
parsed_rows: List[List[str]] = []
for line in lines:
if not line.strip():
continue
row_fields = []
current_field = []
in_quotes = False
i = 0
n = len(line)
while i < n:
char = line[i]
if in_quotes:
if char == '"':
# Lookahead for escaped quote
if i + 1 < n and line[i+1] == '"':
current_field.append('"')
i += 2
continue
else:
in_quotes = False
else:
current_field.append(char)
else:
if char == '"':
in_quotes = True
elif char == delimiter:
row_fields.append("".join(current_field))
current_field = []
else:
current_field.append(char)
i += 1
row_fields.append("".join(current_field))
parsed_rows.append(row_fields)
if not parsed_rows:
return [], [], 0, 0
headers = parsed_rows[0]
data_rows = parsed_rows[1:] if len(parsed_rows) > 1 else []
col_count = len(headers)
row_count = len(parsed_rows)
return headers, data_rows, row_count, col_count
def detect_dominant_delimiter(text: str) -> str:
"""
Analyzes lines to guess if the delimiter is comma, semicolon, or tab.
Looks for the highest count with consistency across initial lines.
"""
candidates = [",", ";", "\t"]
sample_lines = [l for l in text.splitlines()[:5] if l.strip()]
if not sample_lines:
return ","
best_delim = ","
best_score = -1
for delim in candidates:
counts = [line.count(delim) for line in sample_lines]
avg_count = sum(counts) / len(counts)
# Minimize standard deviation for structural consistency
variance = sum((c - avg_count)**2 for c in counts) / len(counts)
if avg_count > 0.5:
score = avg_count / (1.0 + variance) # high frequency, low variation
if score > best_score:
best_score = score
best_delim = delim
return best_delim
# =====================================================================
# TEXT ANALYZER CORE LOGIC
# =====================================================================
class TextAnalyzer:
"""
The monolithic analysis processor wrapping all character,
unicode, word, sentence, regex, and structured analyses.
"""
def __init__(self, text: str):
self.text = text
self.bytes_data = text.encode("utf-8")
# Simple structural tokenizer
self.lines = text.splitlines()
# Word extraction without NLP: lowercase for counts, standard regex boundary matches
self.words_raw = re.findall(r'\b[a-zA-Z0-9_\'-]+\b', text)
self.words = [w.strip() for w in self.words_raw if w.strip()]
# Sentence segmentation heuristic (split on . ! ? followed by space/line bounds)
self.sentences = [s.strip() for s in re.split(r'[.!?]+(?=\s|$)', text) if s.strip()]
def get_overview(self) -> Dict[str, Any]:
"""Module 1: General document measurements."""
char_count = len(self.text)
word_count = len(self.words)
sentence_count = max(1 if word_count > 0 and not self.sentences else 0, len(self.sentences))
line_count = len(self.lines)
# Paragraphs: split by multiple empty lines
paragraphs = [p for p in re.split(r'\n\s*\n', self.text) if p.strip()]
paragraph_count = len(paragraphs)
byte_count = len(self.bytes_data)
avg_word_length = sum(len(w) for w in self.words) / word_count if word_count > 0 else 0.0
avg_sentence_length = word_count / sentence_count if sentence_count > 0 else 0.0
return {
"char_count": char_count,
"word_count": word_count,
"sentence_count": sentence_count,
"line_count": line_count,
"paragraph_count": paragraph_count,
"byte_count": byte_count,
"avg_word_length": round(avg_word_length, 2),
"avg_sentence_length": round(avg_sentence_length, 2),
}
def generate_encoding_table(self, max_chars: int = 150) -> List[List[str]]:
"""
Module 2: Technical character encodings mappings.
Limits rows to avoid crashing browser windows with massive datasets.
"""
rows = []
for char in self.text[:max_chars]:
dec = ord(char)
hex_val = f"0x{dec:X}"
bin_val = f"{dec:08b}"
# ASCII validation
ascii_repr = char if dec < 128 else "N/A"
if dec < 32 or dec == 127:
ascii_repr = "Control Char" if dec != 10 and dec != 9 else ("[LF]" if dec == 10 else "[TAB]")
unicode_point = f"U+{dec:04X}"
# UTF-8 details
utf8_bytes = char.encode("utf-8")
utf8_len = len(utf8_bytes)
utf8_bytes_str = " ".join(f"{b:02X}" for b in utf8_bytes)
rows.append([
char if dec >= 32 else " ",
ascii_repr,
unicode_point,
str(dec),
hex_val,
bin_val,
f"{utf8_len} byte(s)",
utf8_bytes_str
])
return rows
def generate_unicode_explorer(self, max_chars: int = 100) -> str:
"""
Module 3: Educational breakdown highlighting the prefix bit structure of UTF-8.
Provides highly descriptive, interactive visual representation of multibyte UTF-8.
"""
html = ['']
limit_text = self.text[:max_chars]
for idx, char in enumerate(limit_text):
dec_val = ord(char)
cp = f"U+{dec_val:04X}"
utf8_b = char.encode("utf-8")
html.append('
')
# Character Display Panel
char_display = char if dec_val >= 32 else f"
CTRL({dec_val})"
html.append(f'
')
html.append(f'
{char_display}
')
html.append(f'
')
html.append(f'
Character #{idx + 1}
')
html.append(f'
{cp}
')
html.append(f'
')
html.append(f'
')
# Binary byte representation detail
html.append('
')
html.append('
UTF-8 Bit Structure:
')
for b in utf8_b:
bin_str = f"{b:08b}"
# Format markers with color spans according to UTF-8 rule
if len(utf8_b) == 1:
# Single Byte (ASCII): 0xxxxxxx
formatted_bin = f"
0{bin_str[1:]}"
lbl = "ASCII Pattern"
elif len(utf8_b) == 2:
if b == utf8_b[0]:
formatted_bin = f"
110{bin_str[3:]}"
lbl = "2-Byte Header"
else:
formatted_bin = f"
10{bin_str[2:]}"
lbl = "Data Byte"
elif len(utf8_b) == 3:
if b == utf8_b[0]:
formatted_bin = f"
1110{bin_str[4:]}"
lbl = "3-Byte Header"
else:
formatted_bin = f"
10{bin_str[2:]}"
lbl = "Data Byte"
else: # 4-byte sequences
if b == utf8_b[0]:
formatted_bin = f"
11110{bin_str[5:]}"
lbl = "4-Byte Header (Emoji/Rare)"
else:
formatted_bin = f"
10{bin_str[2:]}"
lbl = "Data Byte"
html.append(f'
')
html.append(f' ({lbl})')
html.append(f' {formatted_bin}')
html.append(f'
')
html.append('
') # end byte display
html.append('
') # end card
html.append('
Displaying first {max_chars} characters. Your input has {len(self.text)} total chars.
')
return "\n".join(html)
def get_char_frequencies(self) -> List[Tuple[str, int, float]]:
"""Module 4: Sort and measure character densities."""
total = len(self.text)
if total == 0:
return []
cnt = Counter(self.text)
sorted_chars = cnt.most_common()
return [(c, count, round((count / total) * 100, 2)) for c, count in sorted_chars]
def get_word_frequencies(self) -> List[Tuple[str, int]]:
"""Module 5: Tokenize, clean and compute high vocabulary counts."""
if not self.words:
return []
normalized_words = [w.lower() for w in self.words]
cnt = Counter(normalized_words)
return cnt.most_common()
def get_string_statistics(self) -> Dict[str, Any]:
"""Module 6: Text measurements and distributions."""
unique_words = len(set(w.lower() for w in self.words))
unique_chars = len(set(self.text))
whitespace_count = sum(1 for c in self.text if c.isspace())
tab_count = self.text.count('\t')
newline_count = self.text.count('\n')
digit_count = sum(1 for c in self.text if c.isdigit())
uppercase_count = sum(1 for c in self.text if c.isupper())
lowercase_count = sum(1 for c in self.text if c.islower())
# Punctuation set (Standard English keys + Unicode counterparts)
punct_set = set(string.punctuation) | {'“', '”', '‘', '’', '—', '–', '…', '¿', '¡'}
punctuation_count = sum(1 for c in self.text if c in punct_set)
longest_word = max(self.words, key=len) if self.words else ""
shortest_word = min(self.words, key=len) if self.words else ""
return {
"longest_word": longest_word,
"shortest_word": shortest_word,
"unique_words": unique_words,
"unique_characters": unique_chars,
"whitespace_count": whitespace_count,
"tab_count": tab_count,
"newline_count": newline_count,
"digit_count": digit_count,
"uppercase_count": uppercase_count,
"lowercase_count": lowercase_count,
"punctuation_count": punctuation_count
}
def run_regex_explorer(self) -> Dict[str, List[str]]:
"""
Module 7: Matches semantic groups matching pre-defined structural regex rules.
"""
patterns = {
"Emails": r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}',
"URLs": r'https?://(?:www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b(?:[-a-zA-Z0-9()@:%_\+.~#?&//=]*)',
"Phone numbers": r'\+?[0-9]{1,4}?[-.\s]?(?:\([0-9]{1,3}\)|[0-9]{1,3})[-.\s]?[0-9]{1,4}[-.\s]?[0-9]{1,4}[-.\s]?[0-9]{1,9}',
"Dates": r'\d{4}[-/.]\d{1,2}[-/.]\d{1,2}|\d{1,2}[-/.]\d{1,2}[-/.]\d{2,4}',
"Times": r'\b(?:[01]?\d|2[0-3]):[0-5]\d(?::[0-5]\d)?\s?(?:AM|PM|am|pm)?\b',
"Numbers": r'\b\d+(?:\.\d+)?\b',
"Currency values": r'(?:\$|€|£|¥|₹)\s?\d+(?:,\d{3})*(?:\.\d+)?\b|\b\d+(?:\.\d+)?\s?(?:USD|EUR|GBP|JPY|INR)\b',
"Hashtags": r'#[a-zA-Z0-9_]+',
"Mentions": r'@[a-zA-Z0-9_]+',
"IP addresses": r'\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b'
}
matches = {}
for title, pattern in patterns.items():
found = re.findall(pattern, self.text)
# Remove empty strings or simple fragments
matches[title] = [f.strip() for f in found if f.strip()]
return matches
def detect_format(self) -> Tuple[str, str]:
"""
Module 8: Text Format Detection.
Evaluates criteria and supplies natural explaining logic for findings.
"""
txt = self.text.strip()
if not txt:
return "Empty Input", "No text provided to detect structural formats."
# JSON detection
try:
json.loads(txt)
return "JSON (Valid)", "Parsed cleanly into key-value trees or arrays using standard python serialization (json.loads)."
except json.JSONDecodeError:
if (txt.startswith('{') and txt.endswith('}')) or (txt.startswith('[') and txt.endswith(']')):
return "JSON (Malformed)", "Begins/ends with brackets ({}, []), but contains syntactic errors like misplaced commas or string literals."
# XML detection
try:
ET.fromstring(txt)
return "XML (Valid)", "Parsed successfully using standard ElementTree XML architecture."
except ET.ParseError:
if re.search(r'<\?xml', txt, re.I) or (txt.startswith('<') and txt.endswith('>')):
return "XML (Malformed)", "Starts/ends with tags or contains XML headers, but fails parser schemas."
# HTML detection
html_sig_tags = r']*)*>', txt)
if re.search(html_sig_tags, txt, re.I) or len(tags_found) > 4:
return "HTML", f"Contains signature tags (matched {len(tags_found)} raw HTML elements) or document structures like ''."
# Markdown detection
md_points = 0
reasons = []
if re.search(r'^(?:#|##|###|####|#####|######)\s+.+', txt, re.M):
md_points += 2
reasons.append("Contains structural section indicators (# Header)")
if re.search(r'\[.+?\]\(https?://.+?\)', txt):
md_points += 2
reasons.append("Identified Markdown link structures: [text](URL)")
if re.search(r'^[*-]\s+\w+', txt, re.M):
md_points += 1
reasons.append("Identified bullet lists (- or *)")
if re.search(r'^```\w*\n', txt, re.M):
md_points += 3
reasons.append("Identified structural code fencing tags (```)")
if md_points >= 3:
return "Markdown", f"Identified markdown layout markers: {', '.join(reasons)}."
# CSV/TSV detection
delim = detect_dominant_delimiter(self.text)
lines_with_delim = [l for l in self.text.splitlines() if delim in l]
if len(lines_with_delim) >= 2:
# Check consistency of delimiter count
counts = [l.count(delim) for l in lines_with_delim[:5]]
avg_count = sum(counts) / len(counts)
variance = sum((c - avg_count)**2 for c in counts) / len(counts)
if avg_count > 0 and variance < 1.5:
delim_name = "Comma" if delim == "," else ("Semicolon" if delim == ";" else "Tab")
return f"CSV / Delimited Text", f"Structured grid properties detected using consistently spaced delimiter: '{delim_name}' (average spacing density: {avg_count:.1f} per line)."
return "Plain Text (General)", "Default categorization. Contains no specialized programmatic structure, markup schemas, or clear delimiters."
# =====================================================================
# UI GENERATION HELPER FUNCTIONS
# =====================================================================
def analyze_all_inputs(text: str) -> List[Any]:
"""
Main callback updating all visual components in Gradio blocks concurrently.
"""
if not text or not text.strip():
# Fallback values
empty_overview_html = "Please supply valid text or load a sample.
"
empty_tbl = []
return [
empty_overview_html, empty_tbl, empty_overview_html, empty_tbl, empty_tbl,
empty_overview_html, empty_overview_html, empty_overview_html, empty_overview_html, ""
]
analyzer = TextAnalyzer(text)
# Overview
ov = analyzer.get_overview()
ov_html = f"""
Characters
Words
Sentences
Bytes (UTF-8)
Lines
{ov['line_count']:,}
Paragraphs
{ov['paragraph_count']:,}
Avg Word Length
{ov['avg_word_length']} chars
Avg Sentence Length
{ov['avg_sentence_length']} words
"""
# Visual Charts using Tailwind (Avoid high load graphics libraries)
chars_stat = analyzer.get_string_statistics()
total_measurable = max(1, chars_stat["uppercase_count"] + chars_stat["lowercase_count"] + chars_stat["digit_count"] + chars_stat["punctuation_count"] + chars_stat["whitespace_count"])
def pct(val):
return (val / total_measurable) * 100
overview_chart_html = f"""
Character Class Distribution Ratio:
Lowercase ({chars_stat['lowercase_count']})
Uppercase ({chars_stat['uppercase_count']})
Digits ({chars_stat['digit_count']})
Punctuation ({chars_stat['punctuation_count']})
Whitespace ({chars_stat['whitespace_count']})
"""
# Encodings Table
encodings_data = analyzer.generate_encoding_table()
# Unicode Inspector
unicode_html = analyzer.generate_unicode_explorer()
# Frequencies
char_freqs = [[repr(c)[1:-1] if c != '\n' else '[LF]', count, f"{pct:.1f}%"] for c, count, pct in analyzer.get_char_frequencies()]
word_freqs = [[word, count] for word, count in analyzer.get_word_frequencies()[:100]]
# Detailed statistics mappings
stats_data = [
["Property Metric", "Value", "Educational Context"],
["Longest word length", f"{len(chars_stat['longest_word'])} chars ('{chars_stat['longest_word']}')" if chars_stat['longest_word'] else "N/A", "Useful for finding outlier anomalies or unspaced strings."],
["Shortest word length", f"{len(chars_stat['shortest_word'])} chars ('{chars_stat['shortest_word']}')" if chars_stat['shortest_word'] else "N/A", "Averages lower for basic grammar prepositions."],
["Unique vocabulary (Words)", str(chars_stat['unique_words']), "Vocabulary density indicator before lemmatization / stemming."],
["Unique character keys", str(chars_stat['unique_characters']), "Alphabet size of current document schema."],
["Whitespace instances", str(chars_stat['whitespace_count']), "Sum of space, newline, tab occurrences."],
["Tab character counts", str(chars_stat['tab_count']), "Important identifier for tab-separated grids or indent properties."],
["Line feeds (Newlines)", str(chars_stat['newline_count']), "Tells us structural grouping spacing parameters."],
["Number instances", str(chars_stat['digit_count']), "Shows the count of raw numerals."],
["Uppercase keys", str(chars_stat['uppercase_count']), "Casing properties indicating structural sentences or nouns."],
["Lowercase keys", str(chars_stat['lowercase_count']), "Standard core text body."],
["Punctuation marks", str(chars_stat['punctuation_count']), "Sentence partitions and code punctuation keys."]
]
# Regex Matches HTML Builder
regex_matches = analyzer.run_regex_explorer()
regex_html = ['']
for category, list_matches in regex_matches.items():
count = len(list_matches)
color = "emerald" if count > 0 else "slate"
badge_class = f"bg-{color}-100 text-{color}-800 dark:bg-{color}-950 dark:text-{color}-300"
regex_html.append(f'
')
regex_html.append(f'
')
regex_html.append(f'
{category}
')
regex_html.append(f' {count} matches')
regex_html.append(f' ')
if count > 0:
# Highlight items in tags
items_html = " ".join(f'
{m}' for m in set(list_matches[:30]))
if len(list_matches) > 30:
items_html += f'
...and {len(list_matches)-30} more'
regex_html.append(f'
{items_html}
')
else:
regex_html.append(f'
No matches detected in this pattern configuration.
')
regex_html.append('
')
regex_html.append('
{format_type}
{format_desc}
"""
# Sub-Inspectors (CSV, HTML)
csv_html_str = "CSV structures not detected. This explorer will display formatting tables if text looks tabular.
"
if "CSV" in format_type or "," in text or ";" in text or "\t" in text:
delim = detect_dominant_delimiter(text)
try:
headers, data_rows, r_cnt, c_cnt = parse_csv_custom(text, delim)
if r_cnt > 0:
# Limit row views
preview_rows = data_rows[:10]
rows_html = "".join(f"" + "".join(f"| {col} | " for col in row) + "
" for row in preview_rows)
headers_html = "".join(f"{h} | " for h in headers)
csv_html_str = f"""
Parser Output: {r_cnt} Rows × {c_cnt} Columns
Delimiter: '{delim}'
{headers_html}
{rows_html}
{f'
Showing top 10 preview rows
' if len(data_rows) > 10 else ''}
"""
except Exception as e:
csv_html_str = f"Failed standard CSV matrix processing: {str(e)}
"
# HTML parsing extraction using SimpleHTMLInspector
html_details_str = "HTML markup pattern matches not found.
"
if "HTML" in format_type or "<" in text:
try:
parser = SimpleHTMLInspector()
parser.feed(text)
tag_counts = Counter(parser.tags)
tags_stat_html = " ".join(f"<{tag}> ({cnt})" for tag, cnt in tag_counts.items())
# Content mapping lists
content_preview_html = "".join(f"<{tag}>{data}
" for tag, data in parser.content_map[:15])
html_details_str = f"""
Tag Counts (Structure Streams)
{tags_stat_html if tags_stat_html else 'No tag keys found.'}
Tag Text Extraction Preview
{content_preview_html if content_preview_html else '
No clean text stream mapped inside tags.
'}
{f'
Displaying first 15 mapped nodes
' if len(parser.content_map) > 15 else ''}
"""
except Exception as e:
html_details_str = f"HTML parser trace error: {str(e)}
"
return [
ov_html,
encodings_data,
unicode_html,
char_freqs,
word_freqs,
stats_data,
regex_html_str,
format_html_str,
csv_html_str,
html_details_str,
overview_chart_html
]
def handle_file_upload(file_obj) -> str:
"""
Reads the file path securely and returns decoded text.
Handles decoding failures gracefully by falling back to errors or latin-1.
"""
if file_obj is None:
return ""
try:
# File object passed is a tempfile wrapper path in Gradio
with open(file_obj.name, "rb") as f:
bytes_content = f.read()
try:
return bytes_content.decode("utf-8")
except UnicodeDecodeError:
# Fallback to Latin-1 encoding to preserve characters cleanly
return bytes_content.decode("latin-1")
except Exception as e:
return f"File load error: {str(e)}"
def generate_report(text: str) -> str:
"""
Fulfills Module 11: Export downloadable analysis reports.
Formats analysis sections into a beautifully structured Markdown report file.
"""
if not text or not text.strip():
# Fallback empty path
temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".md")
temp_file.write("# No analysis data available\nPlease submit text first.".encode('utf-8'))
temp_file.close()
return temp_file.name
analyzer = TextAnalyzer(text)
ov = analyzer.get_overview()
chars_stat = analyzer.get_string_statistics()
fmt, desc = analyzer.detect_format()
regex_matches = analyzer.run_regex_explorer()
char_freqs = analyzer.get_char_frequencies()[:20]
word_freqs = analyzer.get_word_frequencies()[:20]
report = []
report.append("# Text Explorer Analysis Report")
report.append(f"Generated on: 2026-07-05 (System Analysis)\n")
report.append("---")
report.append("## 1. Document Overview Statistics")
report.append(f"- **Characters (Count)**: {ov['char_count']}")
report.append(f"- **Words (Count)**: {ov['word_count']}")
report.append(f"- **Sentences**: {ov['sentence_count']}")
report.append(f"- **Line count**: {ov['line_count']}")
report.append(f"- **Paragraphs**: {ov['paragraph_count']}")
report.append(f"- **Bytes (Size)**: {ov['byte_count']} bytes")
report.append(f"- **Average Word Length**: {ov['avg_word_length']} characters")
report.append(f"- **Average Sentence Length**: {ov['avg_sentence_length']} words\n")
report.append("## 2. Text Format Classification")
report.append(f"- **Detected Structure Format**: {fmt}")
report.append(f"- **Deduction Reason**: {desc}\n")
report.append("## 3. String & Lexical Statistics")
report.append(f"- **Longest word**: '{chars_stat['longest_word']}'")
report.append(f"- **Shortest word**: '{chars_stat['shortest_word']}'")
report.append(f"- **Unique words**: {chars_stat['unique_words']}")
report.append(f"- **Unique characters**: {chars_stat['unique_characters']}")
report.append(f"- **Whitespaces total**: {chars_stat['whitespace_count']}")
report.append(f"- **Tabs**: {chars_stat['tab_count']}")
report.append(f"- **Newlines**: {chars_stat['newline_count']}")
report.append(f"- **Numerics (Digits)**: {chars_stat['digit_count']}")
report.append(f"- **Uppercase characters**: {chars_stat['uppercase_count']}")
report.append(f"- **Lowercase characters**: {chars_stat['lowercase_count']}")
report.append(f"- **Punctuation keys**: {chars_stat['punctuation_count']}\n")
report.append("## 4. Top 20 Character Frequencies")
report.append("| Character | Count | Percentage |")
report.append("| :--- | :--- | :--- |")
for char, cnt, pct in char_freqs:
c_repr = repr(char)[1:-1] if char != '\n' else '[LF]'
report.append(f"| `{c_repr}` | {cnt} | {pct:.1f}% |")
report.append("\n")
report.append("## 5. Top 20 Vocabulary Word Frequencies")
report.append("| Word | Count |")
report.append("| :--- | :--- |")
for word, cnt in word_freqs:
report.append(f"| {word} | {cnt} |")
report.append("\n")
report.append("## 6. Regex Inspector Discoveries")
for key, items in regex_matches.items():
if items:
report.append(f"- **{key}** ({len(items)} found): {', '.join(set(items[:15]))}")
else:
report.append(f"- **{key}**: 0 instances detected.")
report.append("\n---\n*Report generated by Text Explorer Space (Pure Python Educational Analyzer).*")
# Write out to temporary local file path
temp_dir = tempfile.gettempdir()
file_path = os.path.join(temp_dir, "text_explorer_report.md")
with open(file_path, "w", encoding="utf-8") as f:
f.write("\n".join(report))
return file_path
# =====================================================================
# CUSTOM CSS STYLE DEFINITIONS FOR THE GRADIO THEME
# =====================================================================
CSS = """
body {
background-color: #f8fafc;
}
.gradio-container {
font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif;
}
.header-hero {
background: linear-gradient(135deg, #4f46e5 0%, #2563eb 100%);
color: white !important;
border-radius: 1rem;
padding: 2.5rem 2rem;
margin-bottom: 2rem;
text-align: center;
box-shadow: 0 4px 6px -1px rgb(0 0 0 / 0.1), 0 2px 4px -2px rgb(0 0 0 / 0.1);
}
.header-hero h1 {
color: white !important;
font-weight: 900 !important;
letter-spacing: -0.025em;
font-size: 2.25rem !important;
margin-bottom: 0.5rem;
}
.header-hero p {
color: #e0e7ff !important;
font-size: 1rem;
}
.educational-card {
background-color: #f0fdfa;
border: 1px solid #ccfbf1;
border-radius: 0.75rem;
padding: 1rem;
margin-bottom: 1rem;
}
.educational-card h4 {
color: #0f766e !important;
margin-top: 0 !important;
}
.educational-card p, .educational-card li {
color: #115e59 !important;
font-size: 0.875rem !important;
}
"""
# =====================================================================
# SAMPLE TEXTSETS FOR CONVENIENT USER EXPERIENCES
# =====================================================================
SAMPLES = {
"Simple English": "Hello World! Feel free to paste any text here to test the NLP Text Explorer system. It works immediately without external libraries.",
"Multilingual & Emoji": "Multilingual test: Bonjour, こんにちは, 안녕하세요! Here is an emoji breakdown to inspect UTF-8 bytes: 😊 🚀 🍕. What is their representation in binary memory space?",
"HTML document sample": '\n\n\n Sample Sandbox\n\n\n \n
Welcome to Text Explorer!
\n
This is standard markup designed to verify our pure HTML-Parser extraction utilities.
\n
Visit our project\n
How Computers Partition Text
Before an NLP model can comprehend context, the document must be mathematically measured:
- Bytes: The actual digital footprint in physical memory (UTF-8 allocates 1 to 4 bytes per character).
- Words & Sentences: Fundamental logic nodes. Without standard tools like NLTK/spaCy, computers extract tokens using punctuation borders and boundary regex.
""")
overview_html = gr.HTML("Analyze text to generate document insights.
")
overview_chart = gr.HTML()
# Character Encodings
with gr.TabItem("🔢 Encodings Table"):
with gr.Accordion("🏫 Educational Spotlight: Encodings Simplified", open=False):
gr.HTML("""
Encoding Architecture Explained
Every character is assigned a unique tracking coordinate:
- ASCII: Traditional 7-bit system spanning decimal positions 0 through 127. Standard English keys only.
- Unicode: A universal catalog assigning identical codepoint indices (e.g., U+1F60A) for all global languages and emojis.
- UTF-8: An optimized encoding translating those codepoints into binary arrays, using smaller slices (1 byte) for English and wider slices for eastern languages or emojis.
""")
encoding_table = gr.DataFrame(
headers=["Char", "ASCII Representation", "Unicode Point", "Decimal", "Hexadecimal", "Binary Value", "UTF-8 Length", "UTF-8 Bytes"],
datatype=["str", "str", "str", "str", "str", "str", "str", "str"],
wrap=True
)
# Unicode Explorer
with gr.TabItem("🌌 Unicode Explorer"):
with gr.Accordion("🎓 Educational Spotlight: Bit Patterns of UTF-8", open=False):
gr.HTML("""
Deciphering the UTF-8 Multi-byte Rules
UTF-8 dynamically sizes bytes based on prefix triggers. Inspect the binary tags below:
- 0xxxxxxx (1 Byte): Matches normal English ASCII indices seamlessly.
- 110xxxxx 10xxxxxx (2 Bytes): Directs unicode sequences (e.g. Greek / Cyrillic).
- 1110xxxx 10xxxxxx 10xxxxxx (3 Bytes): Covers general Asian characters (CJK blocks).
- 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx (4 Bytes): Represents deep emoticons and rare historical glyphs.
""")
unicode_explorer_html = gr.HTML("Displaying individual unicode bit structures...
")
# Frequencies
with gr.TabItem("📈 Frequencies"):
with gr.Row():
with gr.Column():
gr.Markdown("#### Character Frequency Analysis")
char_freq_table = gr.DataFrame(
headers=["Character Key", "Occurrence", "Ratio"],
datatype=["str", "number", "str"]
)
with gr.Column():
gr.Markdown("#### Word Frequency Analysis (Vocabulary)")
word_freq_table = gr.DataFrame(
headers=["Lowercased Token", "Occurrence"],
datatype=["str", "number"]
)
# Advanced Statistics
with gr.TabItem("🧮 Detailed Statistics"):
stats_table = gr.DataFrame(
headers=["Property Metric", "Value", "Educational Context"],
datatype=["str", "str", "str"],
wrap=True
)
# Regex Explorer
with gr.TabItem("🔎 Regex Explorer"):
with gr.Accordion("🧠 Educational Spotlight: Structured Pattern Matching", open=False):
gr.HTML("""
Regex: The Structural Extraction Language
Regular expressions look for matching text layouts. They are extremely effective before models compile lexical vectors:
- Emails: Matches user accounts bounded by domains (
username@host.ext).
- IP Addresses: Recognizes IPv4 address streams (digits separated by decimal boundaries).
""")
regex_explorer_html = gr.HTML("Matches category analysis lists...
")
# Format Detectors
with gr.TabItem("🧱 Layout Formats"):
with gr.Accordion("📦 Educational Spotlight: Layout Analysis Systems", open=False):
gr.HTML("""
Markup and Matrix Structure Detection Rules
Computers parse document systems using distinctive delimiters and structural tags:
- HTML/XML: Uses bracket boundaries (
<tag>) to represent nested elements.
- CSV Grid Structure: Uses consistent grid markers (such as commas) containing identical segments per line row.
""")
format_overview_html = gr.HTML()
with gr.Accordion("CSV Preview Engine (Non-Library Machine)", open=True):
csv_preview_html = gr.HTML("No CSV parsed grid display loaded yet.
")
with gr.Accordion("HTML Tree Parser Detail", open=True):
html_preview_html = gr.HTML("No HTML tag analysis details loaded yet.
")
# Download Export
with gr.TabItem("💾 Export Report"):
gr.Markdown("### Create Inspection Summary File")
gr.Markdown("Download a complete, beautifully structured Markdown text report mapping the calculations of the explorer for external study.")
report_btn = gr.Button("🔨 Compile Markdown Analysis Report", variant="primary")
report_file = gr.File(label="Downloadable Analysis Report", type="filepath")
# =====================================================================
# CONTROLLER WIREFRAMES & ACTIONS
# =====================================================================
# Upload handler updating input text
file_upload.change(
fn=handle_file_upload,
inputs=file_upload,
outputs=input_text
)
# Core reactive interface processing all panels together
ui_outputs = [
overview_html,
encoding_table,
unicode_explorer_html,
char_freq_table,
word_freq_table,
stats_table,
regex_explorer_html,
format_overview_html,
csv_preview_html,
html_preview_html,
overview_chart
]
# Input changes trigger analytical updates instantly
input_text.change(
fn=analyze_all_inputs,
inputs=input_text,
outputs=ui_outputs
)
# Export report trigger
report_btn.click(
fn=generate_report,
inputs=input_text,
outputs=report_file
)
# Run the Gradio Space application
if __name__ == "__main__":
demo.launch()