clear cache

app.py

@@ -23,10 +23,6 @@ def clear_huggingface_cache():
         print("Cache directory does not exist.")
 
 
-# Example usage
-clear_huggingface_cache()
-
-
 # Disable caching globally for Hugging Face datasets
 disable_caching()
 
@@ -55,13 +51,13 @@ def get_data():
     Uses streaming to avoid loading the entire dataset into memory at once,
     which is useful for handling large datasets.
     """
+    clear_huggingface_cache()
     ds = load_dataset(
         DS_NAME,
         streaming=True,
     )
     for row in ds["train"]:
         yield row
-    clear_huggingface_cache()
 
 
 def process_and_push_data():
@@ -77,7 +73,6 @@ def process_and_push_data():
     ds_processed.push_to_hub(TARGET_REPO)
 
     logger.info("Data processed and pushed to the hub.")
-    clear_huggingface_cache()
 
 
 # Initialize the WebhooksServer with Gradio interface (if needed)