youssefreda9 commited on
Commit
075fdd7
·
1 Parent(s): 1e65c93

Fix spacing corruption block to include emails and english words

Browse files
src/nlp/punctuation/punctuation_rules.py CHANGED
@@ -183,7 +183,7 @@ def validate_punctuation_diff(diff: dict, full_text: str = '') -> bool:
183
  return False
184
 
185
  # Block spacing corruptions in JSON/Code (e.g. {"name"} -> { "name" })
186
- if re.search(r'\{|\[|<|://', original):
187
  # Only allow if the ONLY change is appending a terminal mark at the very end
188
  if original != correction and not (is_at_end and correction.endswith(('.', '؟')) and correction[:-1].rstrip() == original.rstrip()):
189
  logger.info(f"[PUNC-SAFETY] Blocked corruption of JSON/Code/URL: '{original}' -> '{correction}'")
 
183
  return False
184
 
185
  # Block spacing corruptions in JSON/Code (e.g. {"name"} -> { "name" })
186
+ if re.search(r'[a-zA-Z]|\{|\[|<|#|@|://', original):
187
  # Only allow if the ONLY change is appending a terminal mark at the very end
188
  if original != correction and not (is_at_end and correction.endswith(('.', '؟')) and correction[:-1].rstrip() == original.rstrip()):
189
  logger.info(f"[PUNC-SAFETY] Blocked corruption of JSON/Code/URL: '{original}' -> '{correction}'")