cache parquet in local duckdb table; tidy slider styling

Load only the columns we actually query into a module-level DuckDB
in-memory table on startup, so filter changes hit the local table
instead of re-fetching the parquet over httpfs every time. Group the
slider, range display, and unknown-params checkbox into one card and
clean up the range-display background.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

app.py

@@ -2,6 +2,8 @@ import gradio as gr
 import pandas as pd
 import plotly.express as px
 import time
+import threading
+import html
 import duckdb
 from huggingface_hub import list_repo_files
 # Using the stable, community-built RangeSlider component
@@ -17,25 +19,75 @@ HF_DATASET_ID = "evijit/modelverse_daily_data"
 TAG_FILTER_CHOICES = [ "Audio & Speech", "Time series", "Robotics", "Music", "Video", "Images", "Text", "Biomedical", "Sciences" ]
 PIPELINE_TAGS = [ 'text-generation', 'text-to-image', 'text-classification', 'text2text-generation', 'audio-to-audio', 'feature-extraction', 'image-classification', 'translation', 'reinforcement-learning', 'fill-mask', 'text-to-speech', 'automatic-speech-recognition', 'image-text-to-text', 'token-classification', 'sentence-similarity', 'question-answering', 'image-feature-extraction', 'summarization', 'zero-shot-image-classification', 'object-detection', 'image-segmentation', 'image-to-image', 'image-to-text', 'audio-classification', 'visual-question-answering', 'text-to-video', 'zero-shot-classification', 'depth-estimation', 'text-ranking', 'image-to-video', 'multiple-choice', 'unconditional-image-generation', 'video-classification', 'text-to-audio', 'time-series-forecasting', 'any-to-any', 'video-text-to-text', 'table-question-answering' ]
 
+# Columns we actually read from the parquet. Projecting at load time keeps the
+# in-memory table small and avoids paying for tag arrays / safetensors blobs.
+NEEDED_COLUMNS = [
+    "id", "organization",
+    "downloads", "downloadsAllTime", "likes",
+    "params", "createdAt", "pipeline_tag", "data_download_timestamp",
+    "is_audio_speech", "has_music", "has_robot", "is_biomed",
+    "has_series", "has_science", "has_video", "has_image", "has_text",
+]
+
+_db_lock = threading.Lock()
+_db_con = None
+_data_stats = None
+
+
+def get_db():
+    return _db_con
+
 
 def load_models_data():
+    """Download the parquet once into a local DuckDB in-memory table.
+
+    Subsequent calls are no-ops — the cached connection is reused for every
+    plot regeneration, so filter changes never re-hit the network.
+    """
+    global _db_con, _data_stats
     overall_start_time = time.time()
-    print(f"Attempting to load dataset metadata from Hugging Face Hub: {HF_DATASET_ID}")
-    try:
-        files = list_repo_files(HF_DATASET_ID, repo_type="dataset")
-        parquet_files = [f for f in files if f.endswith('.parquet')]
-        if not parquet_files:
-            return [], False, "No parquet files found in dataset."
-            
-        urls = [f"https://huggingface.co/datasets/{HF_DATASET_ID}/resolve/main/{f}" for f in parquet_files]
-        
-        msg = f"Successfully identified {len(urls)} parquet files in {time.time() - overall_start_time:.2f}s."
-        print(msg)
-        return urls, True, msg
-    except Exception as e:
-        err_msg = f"Failed to load dataset metadata. Error: {e}"
-        print(err_msg)
-        return [], False, err_msg
+    with _db_lock:
+        if _db_con is not None:
+            return True, "Data already loaded.", _data_stats
+
+        print(f"Attempting to load dataset from Hugging Face Hub: {HF_DATASET_ID}")
+        try:
+            files = list_repo_files(HF_DATASET_ID, repo_type="dataset")
+            parquet_files = [f for f in files if f.endswith('.parquet')]
+            if not parquet_files:
+                return False, "No parquet files found in dataset.", None
+
+            urls = [f"https://huggingface.co/datasets/{HF_DATASET_ID}/resolve/main/{f}" for f in parquet_files]
+            urls_str = ", ".join([f"'{u}'" for u in urls])
+            cols_str = ", ".join(NEEDED_COLUMNS)
+
+            con = duckdb.connect()
+            con.execute("INSTALL httpfs; LOAD httpfs;")
+            con.execute(
+                f"CREATE TABLE models AS SELECT {cols_str} FROM read_parquet([{urls_str}])"
+            )
+
+            stats_row = con.execute(
+                "SELECT count(*), max(data_download_timestamp), count(params) FROM models"
+            ).fetchone()
+            _data_stats = {
+                "total_count": stats_row[0],
+                "timestamp": stats_row[1],
+                "param_count": stats_row[2],
+                "num_files": len(urls),
+            }
+            _db_con = con
+
+            msg = (
+                f"Loaded {stats_row[0]:,} models from {len(urls)} parquet file(s) "
+                f"in {time.time() - overall_start_time:.2f}s."
+            )
+            print(msg)
+            return True, msg, _data_stats
+        except Exception as e:
+            err_msg = f"Failed to load dataset. Error: {e}"
+            print(err_msg)
+            return False, err_msg, None
 
 def get_param_range_values(param_range_labels):
     min_label, max_label = param_range_labels
@@ -43,15 +95,11 @@ def get_param_range_values(param_range_labels):
     max_val = float('inf') if '>' in max_label else float(max_label.replace('B', ''))
     return min_val, max_val
 
-def make_treemap_data(parquet_urls, count_by, top_k=25, tag_filter=None, pipeline_filter=None, param_range=None, skip_orgs=None, include_unknown_param_size=True, created_after_date: float = None):
-    if not parquet_urls: return pd.DataFrame()
-    
-    con = duckdb.connect()
-    con.execute("INSTALL httpfs; LOAD httpfs;")
-    
-    urls_str = ", ".join([f"'{u}'" for u in parquet_urls])
-    con.execute(f"CREATE VIEW models AS SELECT * FROM read_parquet([{urls_str}])")
-    
+def make_treemap_data(count_by, top_k=25, tag_filter=None, pipeline_filter=None, param_range=None, skip_orgs=None, include_unknown_param_size=True, created_after_date: float = None):
+    con = get_db()
+    if con is None:
+        return pd.DataFrame()
+
     where_clauses = []
     
     if not include_unknown_param_size:
@@ -127,19 +175,32 @@ def create_treemap(treemap_data, count_by, title=None):
     return fig
 
 custom_css = """
-.model-parameters-group > .block {
-    background: none !important;
+/* Hide the RangeSlider's built-in value display bits — the small number
+   bubbles above the track, and the min/max number inputs + reset button
+   inside `.head`. We keep the label itself (the first child of `.head`). */
+#param-slider-wrapper div[data-testid="range-slider"] > span {
+    display: none !important;
+}
+#param-slider-wrapper .head > *:not(:first-child) {
+    display: none !important;
+}
+.param-range-display,
+.param-range-display * {
+    background: var(--background-fill-primary) !important;
+    background-color: var(--background-fill-primary) !important;
     border: none !important;
     box-shadow: none !important;
 }
-#param-slider-wrapper .head,
-#param-slider-wrapper div[data-testid="range-slider"] > span {
-    display: none !important;
+.param-range-display {
+    padding: 0.5rem 0.75rem !important;
+}
+.param-range-display p {
+    font-size: 0.95rem !important;
+    margin: 0 !important;
 }
 """
 
 with gr.Blocks(title="🤗 ModelVerse Explorer", fill_width=True, css=custom_css) as demo:
-    models_data_state = gr.State([])
     loading_complete_state = gr.State(False)
     
     with gr.Row():
@@ -154,13 +215,15 @@ with gr.Blocks(title="🤗 ModelVerse Explorer", fill_width=True, css=custom_css
             tag_filter_dropdown = gr.Dropdown(label="Select Tag", choices=TAG_FILTER_CHOICES, value=None, visible=False)
             pipeline_filter_dropdown = gr.Dropdown(label="Select Pipeline Tag", choices=PIPELINE_TAGS, value=None, visible=False)
             
-            with gr.Group(elem_classes="model-parameters-group"):
-                gr.Markdown("<div style='font-weight: 500;'>Model Parameters</div>")
+            with gr.Group():
                 param_range_slider = RangeSlider(
                     minimum=0, maximum=len(PARAM_CHOICES) - 1, value=PARAM_CHOICES_DEFAULT_INDICES,
-                    step=1, label=None, show_label=False, elem_id="param-slider-wrapper"
+                    step=1, label="Model Parameters", elem_id="param-slider-wrapper"
+                )
+                param_range_display = gr.Markdown(
+                    f"Selected Range: <b>{html.escape(PARAM_CHOICES[0])}</b> to <b>{html.escape(PARAM_CHOICES[-1])}</b>",
+                    elem_classes="param-range-display",
                 )
-                param_range_display = gr.Markdown(f"Range: `{PARAM_CHOICES[0]}` to `{PARAM_CHOICES[-1]}`")
                 include_unknown_params_checkbox = gr.Checkbox(label="Include models with unknown parameter size", value=True)
             
             created_after_datepicker = gr.DateTime(label="Created After")
@@ -177,7 +240,10 @@ with gr.Blocks(title="🤗 ModelVerse Explorer", fill_width=True, css=custom_css
     
     def update_param_display(value: tuple):
         min_idx, max_idx = int(value[0]), int(value[1])
-        return f"Range: `{PARAM_CHOICES[min_idx]}` to `{PARAM_CHOICES[max_idx]}`"
+        return (
+            f"Selected Range: <b>{html.escape(PARAM_CHOICES[min_idx])}</b> "
+            f"to <b>{html.escape(PARAM_CHOICES[max_idx])}</b>"
+        )
     
     def _toggle_unknown_params_checkbox(param_range_indices):
         min_idx, max_idx = int(param_range_indices[0]), int(param_range_indices[1])
@@ -196,27 +262,16 @@ with gr.Blocks(title="🤗 ModelVerse Explorer", fill_width=True, css=custom_css
                                filter_choice_radio, [tag_filter_dropdown, pipeline_filter_dropdown])
 
     def load_and_generate_initial_plot(progress=gr.Progress()):
-        progress(0, desc=f"Loading dataset metadata '{HF_DATASET_ID}'...")
-        parquet_urls, load_success_flag, status_msg_from_load = [], False, ""
+        progress(0, desc=f"Loading dataset '{HF_DATASET_ID}'...")
+        load_success_flag, status_msg_from_load, stats = False, "", None
         try:
-            parquet_urls, load_success_flag, status_msg_from_load = load_models_data()
-            if load_success_flag:
-                progress(0.5, desc="Processing metadata...")
-                
-                # Quick query to get stats
-                con = duckdb.connect()
-                con.execute("INSTALL httpfs; LOAD httpfs;")
-                urls_str = ", ".join([f"'{u}'" for u in parquet_urls])
-                con.execute(f"CREATE VIEW models AS SELECT * FROM read_parquet([{urls_str}])")
-                
-                # Get total count and timestamp
-                stats = con.execute("SELECT count(*), max(data_download_timestamp), count(params) FROM models").fetchone()
-                total_count = stats[0]
-                ts = stats[1] # Timestamp object
-                param_count = stats[2]
-                
+            load_success_flag, status_msg_from_load, stats = load_models_data()
+            if load_success_flag and stats is not None:
+                ts = stats["timestamp"]
+                total_count = stats["total_count"]
+                param_count = stats["param_count"]
                 date_display = ts.strftime('%B %d, %Y, %H:%M:%S %Z') if ts else "Pre-processed (date unavailable)"
-                
+
                 data_info_text = (f"### Data Information\n- Source: `{HF_DATASET_ID}`\n- Status: {status_msg_from_load}\n"
                                   f"- Total models loaded: {total_count:,}\n- Models with known parameter counts: {param_count:,}\n"
                                   f"- Models with unknown parameter counts: {total_count - param_count:,}\n- Data as of: {date_display}\n")
@@ -229,43 +284,43 @@ with gr.Blocks(title="🤗 ModelVerse Explorer", fill_width=True, css=custom_css
 
         progress(0.6, desc="Generating initial plot...")
         initial_plot, initial_status = ui_generate_plot_controller(
-            "downloads", "None", None, None, PARAM_CHOICES_DEFAULT_INDICES, 25, 
-            "TheBloke,MaziyarPanahi,unsloth,modularai,Gensyn,bartowski", True, None, parquet_urls, progress
+            "downloads", "None", None, None, PARAM_CHOICES_DEFAULT_INDICES, 25,
+            "TheBloke,MaziyarPanahi,unsloth,modularai,Gensyn,bartowski", True, None, progress
         )
-        return parquet_urls, load_success_flag, data_info_text, initial_status, initial_plot
+        return load_success_flag, data_info_text, initial_status, initial_plot
 
-    def ui_generate_plot_controller(metric_choice, filter_type, tag_choice, pipeline_choice, 
+    def ui_generate_plot_controller(metric_choice, filter_type, tag_choice, pipeline_choice,
                                    param_range_indices, k_orgs, skip_orgs_input, include_unknown_param_size_flag,
-                                   created_after_date, parquet_urls, progress=gr.Progress()):
-        if not parquet_urls:
+                                   created_after_date, progress=gr.Progress()):
+        if get_db() is None:
             return create_treemap(pd.DataFrame(), metric_choice, "Error: Model Data Not Loaded"), "Model data is not loaded."
-        
+
         progress(0.1, desc="Preparing data...")
         param_labels = [PARAM_CHOICES[int(param_range_indices[0])], PARAM_CHOICES[int(param_range_indices[1])]]
-        
+
         treemap_df = make_treemap_data(
-            parquet_urls, metric_choice, k_orgs, 
-            tag_choice if filter_type == "Tag Filter" else None, 
+            metric_choice, k_orgs,
+            tag_choice if filter_type == "Tag Filter" else None,
             pipeline_choice if filter_type == "Pipeline Filter" else None,
-            param_labels, [org.strip() for org in skip_orgs_input.split(',') if org.strip()], 
+            param_labels, [org.strip() for org in skip_orgs_input.split(',') if org.strip()],
             include_unknown_param_size_flag, created_after_date
         )
-        
+
         progress(0.7, desc="Generating plot...")
         title_labels = {"downloads": "Downloads (last 30 days)", "downloadsAllTime": "Downloads (All Time)", "likes": "Likes"}
         plotly_fig = create_treemap(treemap_df, metric_choice, f"HuggingFace Models - {title_labels.get(metric_choice, metric_choice)} by Organization")
-        
+
         plot_stats_md = (f"## Plot Statistics\n- **Models shown**: {len(treemap_df['id'].unique()):,}\n"
                          f"- **Total {metric_choice}**: {int(treemap_df[metric_choice].sum()):,}") if not treemap_df.empty else "No data matches the selected filters."
         return plotly_fig, plot_stats_md
 
-    demo.load(load_and_generate_initial_plot, None, [models_data_state, loading_complete_state, data_info_md, status_message_md, plot_output])
+    demo.load(load_and_generate_initial_plot, None, [loading_complete_state, data_info_md, status_message_md, plot_output])
 
     generate_plot_button.click(
         ui_generate_plot_controller,
         [count_by_dropdown, filter_choice_radio, tag_filter_dropdown, pipeline_filter_dropdown,
          param_range_slider, top_k_dropdown, skip_orgs_textbox, include_unknown_params_checkbox,
-         created_after_datepicker, models_data_state],
+         created_after_datepicker],
         [plot_output, status_message_md]
     )