Update app.py

app.py

@@ -3,97 +3,44 @@ import subprocess
 import sys
 import re
 import numpy as np
+from PIL import Image
 import gradio as gr
 import requests
 import json
 from dotenv import load_dotenv
 
+# Attempt to install pytesseract if not found
+try:
+    import pytesseract
+except ImportError:
+    subprocess.check_call([sys.executable, '-m', 'pip', 'install', 'pytesseract'])
+    import pytesseract
+
+# AFTER importing pytesseract, then set the path
+try:
+    # First try the default path
+    if os.path.exists('/usr/bin/tesseract'):
+        pytesseract.pytesseract.tesseract_cmd = '/usr/bin/tesseract'
+    # Try to find it on the PATH
+    else:
+        tesseract_path = subprocess.check_output(['which', 'tesseract']).decode().strip()
+        if tesseract_path:
+            pytesseract.pytesseract.tesseract_cmd = tesseract_path
+except:
+    # If all else fails, try the default installation path
+    pytesseract.pytesseract.tesseract_cmd = 'tesseract'
+
 # Load environment variables
 load_dotenv()
 
 # Mistral API Key
-MISTRAL_API_KEY = os.getenv("MISTRAL_API_KEY", "GlrVCBWyvTYjWGKl5jqtK4K41uWWJ79F") # Retrieve from environment or default
+MISTRAL_API_KEY = os.getenv("MISTRAL_API_KEY", "GlrVCBWyvTYjWGKl5jqtK4K41uWWJ79F")
 
-# Open Food Facts API for product lookup
-def get_product_ingredients(product_name):
-    """
-    Look up a product by name using the Open Food Facts API and return its ingredients
-    """
-    if not product_name or not product_name.strip():
-        return None, "Please enter a product name"
-
-    try:
-        # Search for products matching the name
-        search_url = f"https://world.openfoodfacts.org/cgi/search.pl?search_terms={product_name}&search_simple=1&action=process&json=1"
-        response = requests.get(search_url)
-        
-        if response.status_code != 200:
-            return None, f"Error connecting to food database: {response.status_code}"
-        
-        data = response.json()
-        
-        # Check if any products were found
-        if data["count"] == 0:
-            return None, f"No products found matching '{product_name}'"
-        
-        # Get the first (most relevant) product
-        product = data["products"][0]
-        
-        # Extract product information
-        product_info = {
-            "name": product.get("product_name", "Unknown product"),
-            "brand": product.get("brands", "Unknown brand"),
-            "ingredients_text": product.get("ingredients_text", ""),
-            "image_url": product.get("image_url", ""),
-            "ingredients_list": []
-        }
-        
-        # If ingredients are available in structured format
-        if "ingredients" in product and isinstance(product["ingredients"], list):
-            for ing in product["ingredients"]:
-                if "text" in ing and ing["text"]:
-                    product_info["ingredients_list"].append(ing["text"].lower())
-        
-        # If no structured ingredients but we have text, parse it
-        elif product_info["ingredients_text"]:
-            product_info["ingredients_list"] = parse_ingredients(product_info["ingredients_text"])
-        
-        # If we still don't have ingredients
-        if not product_info["ingredients_list"]:
-            return None, f"Found product '{product_info['name']}' by {product_info['brand']}, but no ingredients information is available"
-        
-        return product_info, None
-        
-    except requests.exceptions.RequestException as e:
-        return None, f"Network error retrieving product information: {str(e)}"
-    except Exception as e:
-        return None, f"Error retrieving product information: {str(e)}"
-
-# Function to parse ingredients from text
-def parse_ingredients(text):
-    if not text:
-        return []
-
-    # Clean up the text
-    text = re.sub(r'^ingredients:?\s*', '', text.lower(), flags=re.IGNORECASE)
-
-    # Remove common OCR errors and extraneous characters
-    text = re.sub(r'[|\\/@#$%^&*()_+=]', '', text)
-
-    # Split by common ingredient separators
-    ingredients = re.split(r',|;|\n', text)
-
-    # Clean up each ingredient
-    cleaned_ingredients = []
-    for i in ingredients:
-        i = i.strip().lower()
-        if i and len(i) > 1:  # Ignore single characters which are likely errors
-            cleaned_ingredients.append(i)
-
-    return cleaned_ingredients
+# OpenAI API Key for Product Identification
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "sk-exampleapikey") # Replace with your actual OpenAI API key
 
 # Import and configure Mistral API
-def analyze_ingredients_with_mistral(ingredients_list, product_name="", health_conditions=None):
+def analyze_ingredients_with_mistral(ingredients_list, health_conditions=None):
     """
     Use Mistral AI to analyze ingredients and provide health insights.
     """
@@ -102,13 +49,11 @@ def analyze_ingredients_with_mistral(ingredients_list, product_name="", health_c
 
     # Prepare the list of ingredients for the prompt
     ingredients_text = ", ".join(ingredients_list)
-    
-    product_context = f"for the product '{product_name}'" if product_name else ""
 
     # Create a prompt for Mistral
     if health_conditions and health_conditions.strip():
         prompt = f"""
-        Analyze the following food ingredients {product_context} for a person with these health conditions: {health_conditions}
+        Analyze the following food ingredients for a person with these health conditions: {health_conditions}
         Ingredients: {ingredients_text}
         For each ingredient:
         1. Provide its potential health benefits
@@ -119,7 +64,7 @@ def analyze_ingredients_with_mistral(ingredients_list, product_name="", health_c
         """
     else:
         prompt = f"""
-        Analyze the following food ingredients {product_context}:
+        Analyze the following food ingredients:
         Ingredients: {ingredients_text}
         For each ingredient:
         1. Provide its potential health benefits
@@ -155,13 +100,11 @@ def analyze_ingredients_with_mistral(ingredients_list, product_name="", health_c
 
         return analysis + disclaimer
 
-    except requests.exceptions.RequestException as e:
-        # Fallback to basic analysis if API call fails
-        return dummy_analyze(ingredients_list, health_conditions) + f"\n\n(Using fallback analysis: Network error - {str(e)})"
     except Exception as e:
         # Fallback to basic analysis if API call fails
         return dummy_analyze(ingredients_list, health_conditions) + f"\n\n(Using fallback analysis: {str(e)})"
 
+
 # Dummy analysis function for when API is not available
 def dummy_analyze(ingredients_list, health_conditions=None):
     ingredients_text = ", ".join(ingredients_list)
@@ -193,55 +136,275 @@ def dummy_analyze(ingredients_list, health_conditions=None):
 
     return report
 
-# Function to process product name input
-def process_product_input(product_name, manual_ingredients, use_manual, health_conditions):
-    if use_manual:
-        if manual_ingredients and manual_ingredients.strip():
-            ingredients = parse_ingredients(manual_ingredients)
-            return f"Analyzing manually entered ingredients:\n\n{analyze_ingredients_with_mistral(ingredients, '', health_conditions)}", manual_ingredients
+# Function to extract text from images using OCR
+def extract_text_from_image(image):
+    try:
+        if image is None:
+            return "No image captured. Please try again."
+
+        # Verify Tesseract executable is accessible
+        try:
+            subprocess.run([pytesseract.pytesseract.tesseract_cmd, "--version"],
+                          check=True, capture_output=True, text=True)
+        except (subprocess.SubprocessError, FileNotFoundError):
+            return "Tesseract OCR is not installed or not properly configured. Please check installation."
+
+        # Import necessary libraries
+        import cv2
+        import numpy as np
+        from PIL import Image, ImageOps, ImageEnhance
+
+        # First approach: Invert the image for light text on dark background
+        inverted_image = ImageOps.invert(image)
+
+        # Try OCR on inverted image
+        custom_config = r'--oem 3 --psm 6 -l eng --dpi 300'
+        inverted_text = pytesseract.image_to_string(inverted_image, config=custom_config)
+
+        # Second approach: OpenCV processing for colored backgrounds
+        img_cv = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
+
+        # Convert to grayscale
+        gray = cv2.cvtColor(img_cv, cv2.COLOR_BGR2GRAY)
+
+        # Apply bilateral filter to preserve edges while reducing noise
+        filtered = cv2.bilateralFilter(gray, 11, 17, 17)
+
+        # Adaptive thresholding to handle varied lighting
+        thresh = cv2.adaptiveThreshold(filtered, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C,
+                                      cv2.THRESH_BINARY, 11, 2)
+
+        # Invert the image (if text is light on dark background)
+        inverted_thresh = cv2.bitwise_not(thresh)
+
+        # Try OCR on processed image
+        cv_text = pytesseract.image_to_string(
+            Image.fromarray(inverted_thresh),
+            config=custom_config
+        )
+
+        # Third approach: Color filtering to isolate text from colored background
+        # Convert to HSV color space to better isolate colors
+        hsv = cv2.cvtColor(img_cv, cv2.COLOR_BGR2HSV)
+
+        # Create a mask to extract light colored text (assuming white/light text)
+        lower_white = np.array([0, 0, 150])
+        upper_white = np.array([180, 30, 255])
+        mask = cv2.inRange(hsv, lower_white, upper_white)
+
+        # Apply morphological operations to clean up the mask
+        kernel = np.ones((2, 2), np.uint8)
+        mask = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, kernel)
+        mask = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel)
+
+        # Improve character connectivity
+        mask = cv2.dilate(mask, kernel, iterations=1)
+
+        # Try OCR on color filtered image
+        color_text = pytesseract.image_to_string(
+            Image.fromarray(mask),
+            config=r'--oem 3 --psm 6 -l eng --dpi 300'
+        )
+
+        # Fourth approach: Try directly with the image but with different configs
+        direct_text = pytesseract.image_to_string(
+            image,
+            config=r'--oem 3 --psm 11 -l eng --dpi 300'
+        )
+
+        # Compare results and select the best one
+        results = [inverted_text, cv_text, color_text, direct_text]
+
+        # Select the result with the most alphanumeric characters
+        def count_alphanumeric(text):
+            return sum(c.isalnum() for c in text)
+
+        best_text = max(results, key=count_alphanumeric)
+
+        # If still poor results, try with explicit text color inversion in tesseract
+        if count_alphanumeric(best_text) < 20:
+            # Try with tesseract's built-in inversion
+            neg_text = pytesseract.image_to_string(
+                image,
+                config=r'--oem 3 --psm 6 -c textord_heavy_nr=1 -c textord_debug_printable=0 -l eng --dpi 300'
+            )
+            if count_alphanumeric(neg_text) > count_alphanumeric(best_text):
+                best_text = neg_text
+
+        # Clean up the text
+        best_text = re.sub(r'[^\w\s,;:%.()\n\'-]', '', best_text)
+        best_text = best_text.replace('\n\n', '\n')
+
+        # Special case for ingredients list format
+        if "ingredient" in best_text.lower() or any(x in best_text.lower() for x in ["sugar", "cocoa", "milk", "contain"]):
+            # Specific cleaning for ingredient lists
+            best_text = re.sub(r'([a-z])([A-Z])', r'\1 \2', best_text)  # Add space between lowercase and uppercase
+            best_text = re.sub(r'(\d+)([a-zA-Z])', r'\1 \2', best_text)  # Add space between number and letter
+
+        if not best_text.strip():
+            return "No text could be extracted. Ensure image is clear and readable."
+
+        return best_text.strip()
+    except Exception as e:
+        return f"Error extracting text: {str(e)}"
+
+# Function to parse ingredients from text
+def parse_ingredients(text):
+    if not text:
+        return []
+
+    # Clean up the text
+    text = re.sub(r'^ingredients:?\s*', '', text.lower(), flags=re.IGNORECASE)
+
+    # Remove common OCR errors and extraneous characters
+    text = re.sub(r'[|\\/@#$%^&*()_+=]', '', text)
+
+    # Replace common OCR errors
+    text = re.sub(r'\bngredients\b', 'ingredients', text)
+
+    # Handle common OCR misreads
+    replacements = {
+        '0': 'o', 'l': 'i', '1': 'i',
+        '5': 's', '8': 'b', 'Q': 'g',
+    }
+
+    for error, correction in replacements.items():
+        text = text.replace(error, correction)
+
+    # Split by common ingredient separators
+    ingredients = re.split(r',|;|\n', text)
+
+    # Clean up each ingredient
+    cleaned_ingredients = []
+    for i in ingredients:
+        i = i.strip().lower()
+        if i and len(i) > 1:  # Ignore single characters which are likely OCR errors
+            cleaned_ingredients.append(i)
+
+    return cleaned_ingredients
+
+def identify_product_and_get_ingredients(image):
+    """
+    Identifies the product from the image using OpenAI and retrieves ingredients.
+    """
+    try:
+        # Encode the image to base64
+        buffered = io.BytesIO()
+        image.save(buffered, format="JPEG")
+        img_str = base64.b64encode(buffered.getvalue()).decode('utf-8')
+
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {OPENAI_API_KEY}"
+        }
+
+        payload = {
+            "model": "gpt-4-vision-preview",
+            "messages": [
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "text",
+                            "text": "Identify the food product in this image.  If identifiable, also find its ingredients."
+                        },
+                        {
+                            "type": "image_url",
+                            "image_url": {
+                                "url": f"data:image/jpeg;base64,{img_str}"
+                            }
+                        }
+                    ]
+                }
+            ],
+            "max_tokens": 500
+        }
+
+        response = requests.post("https://api.openai.com/v1/chat/completions", headers=headers, json=payload)
+        response.raise_for_status()  # Raise HTTPError for bad responses (4xx or 5xx)
+
+        data = response.json()
+        response_text = data['choices'][0]['message']['content']
+
+        # Attempt to extract ingredients from the response
+        ingredients_match = re.search(r"Ingredients:\s*(.+)", response_text, re.IGNORECASE)
+        if ingredients_match:
+            ingredients_text = ingredients_match.group(1)
+            ingredients = parse_ingredients(ingredients_text)
+            return ingredients, response_text
+        else:
+            # If ingredients not found, return the full response for further handling
+            return None, response_text
+
+    except requests.exceptions.RequestException as e:
+        return None, f"Error during OpenAI API request: {e}"
+    except json.JSONDecodeError as e:
+        return None, f"Error decoding JSON response from OpenAI: {e}"
+    except Exception as e:
+        return None, f"Error identifying product: {e}"
+
+# Function to process input based on method (camera, upload, or manual entry)
+def process_input(input_method, text_input, camera_input, upload_input, health_conditions):
+    if input_method == "Camera":
+        if camera_input is not None:
+            ingredients, response_text = identify_product_and_get_ingredients(camera_input)
+
+            if ingredients:
+                return analyze_ingredients_with_mistral(ingredients, health_conditions), response_text
+            else:
+                return f"Could not identify ingredients from the image. Response from OpenAI:\n\n{response_text}", response_text
+
+        else:
+            return "No camera image captured. Please try again.", ""
+
+    elif input_method == "Image Upload":
+        if upload_input is not None:
+            ingredients, response_text = identify_product_and_get_ingredients(upload_input)
+            if ingredients:
+                return analyze_ingredients_with_mistral(ingredients, health_conditions), response_text
+            else:
+                return f"Could not identify ingredients from the image. Response from OpenAI:\n\n{response_text}", response_text
+
+        else:
+            return "No image uploaded. Please try again.", ""
+
+    elif input_method == "Manual Entry":
+        if text_input and text_input.strip():
+            ingredients = parse_ingredients(text_input)
+            return analyze_ingredients_with_mistral(ingredients, health_conditions), ""
         else:
             return "No ingredients entered. Please try again.", ""
-    else:
-        if not product_name or not product_name.strip():
-            return "Please enter a product name", ""
-            
-        product_info, error = get_product_ingredients(product_name)
-        
-        if error:
-            return error, ""
-            
-        # Display analysis using the ingredients
-        return f"# Analysis for {product_info['name']} by {product_info['brand']}\n\n{analyze_ingredients_with_mistral(product_info['ingredients_list'], product_info['name'], health_conditions)}", ", ".join(product_info['ingredients_list'])
+
+    return "Please provide input using one of the available methods.", ""
 
 # Create the Gradio interface
 with gr.Blocks(title="AI Ingredient Scanner") as app:
     gr.Markdown("# AI Ingredient Scanner")
-    gr.Markdown("Find and analyze product ingredients for health benefits, risks, and potential allergens.")
+    gr.Markdown("Scan product ingredients and analyze them for health benefits, risks, and potential allergens.")
 
     with gr.Row():
         with gr.Column():
-            # Option to use manual ingredients or product lookup
-            use_manual_ingredients = gr.Checkbox(
-                label="Enter ingredients manually instead of product lookup",
-                value=False
-            )
-            
-            # Product name input
-            product_name_input = gr.Textbox(
-                label="Enter product name",
-                placeholder="Oreo cookies, Coca Cola, Nutella, etc.",
-                lines=1
+            input_method = gr.Radio(
+                ["Camera", "Image Upload", "Manual Entry"],
+                label="Input Method",
+                value="Camera"
             )
-            
-            # Manual ingredients input (initially hidden)
-            manual_ingredients_input = gr.Textbox(
+
+            # Camera input
+            camera_input = gr.Image(label="Capture image of product", type="pil", visible=True)
+
+            # Image upload
+            upload_input = gr.Image(label="Upload image of product", type="pil", visible=False)
+
+            # Text input
+            text_input = gr.Textbox(
                 label="Enter ingredients list (comma separated)",
                 placeholder="milk, sugar, flour, eggs, vanilla extract",
                 lines=3,
                 visible=False
             )
 
-            # Health conditions input
+            # Health conditions input - now optional and more flexible
             health_conditions = gr.Textbox(
                 label="Enter your health concerns (optional)",
                 placeholder="diabetes, high blood pressure, peanut allergy, etc.",
@@ -253,32 +416,32 @@ with gr.Blocks(title="AI Ingredient Scanner") as app:
 
         with gr.Column():
             output = gr.Markdown(label="Analysis Results")
-            ingredients_output = gr.Textbox(label="Identified Ingredients", lines=3)
+            extracted_text_output = gr.Textbox(label="Extracted Response from OpenAI", lines=5)
 
-    # Show/hide inputs based on checkbox
-    def update_visible_inputs(use_manual):
+    # Show/hide inputs based on selection
+    def update_visible_inputs(choice):
         return {
-            product_name_input: gr.update(visible=not use_manual),
-            manual_ingredients_input: gr.update(visible=use_manual)
+            upload_input: gr.update(visible=(choice == "Image Upload")),
+            camera_input: gr.update(visible=(choice == "Camera")),
+            text_input: gr.update(visible=(choice == "Manual Entry"))
         }
 
-    use_manual_ingredients.change(update_visible_inputs, use_manual_ingredients, [product_name_input, manual_ingredients_input])
+    input_method.change(update_visible_inputs, input_method, [upload_input, camera_input, text_input])
 
     # Set up event handlers
     analyze_button.click(
-        fn=process_product_input,
-        inputs=[product_name_input, manual_ingredients_input, use_manual_ingredients, health_conditions],
-        outputs=[output, ingredients_output]
+        fn=process_input,
+        inputs=[input_method, text_input, camera_input, upload_input, health_conditions],
+        outputs=[output, extracted_text_output]
     )
 
     gr.Markdown("### How to use")
     gr.Markdown("""
-    1. Enter a product name (e.g., "Oreo cookies", "Coca Cola", "Nutella")
-    2. Or check the box to enter ingredients manually if you prefer
+    1. Choose your input method (Camera, Image Upload, or Manual Entry)
+    2. Take a photo of the product or upload an image, or enter ingredients manually
     3. Optionally enter your health concerns
     4. Click "Analyze Ingredients" to get your personalized analysis
-    
-    The AI will automatically find the product's ingredients and analyze their health implications and potential impact on your specific health concerns.
+    The AI will automatically identify the product and analyze the ingredients, their health implications, and their potential impact on your specific health concerns.
     """)
 
     gr.Markdown("### Examples of what you can ask")
@@ -294,9 +457,10 @@ with gr.Blocks(title="AI Ingredient Scanner") as app:
 
     gr.Markdown("### Tips for best results")
     gr.Markdown("""
-    - Use the full product name and brand if possible
+    - Hold the camera steady and ensure good lighting
+    - Focus directly on the product, including its label
+    - Make sure the product is clearly visible
     - Be specific about your health concerns for more targeted analysis
-    - If a product can't be found, try entering the ingredients manually
     """)
 
     gr.Markdown("### Disclaimer")
@@ -307,4 +471,6 @@ with gr.Blocks(title="AI Ingredient Scanner") as app:
 
 # Launch the app
 if __name__ == "__main__":
+    import io
+    import base64
     app.launch()