import gradio as gr from codecarbon import EmissionsTracker import os import json from datetime import datetime import requests from huggingface_hub import HfApi import tempfile from dotenv import load_dotenv # Load environment variables load_dotenv() # Get environment variables HF_TOKEN = os.getenv("HF_TOKEN") if not HF_TOKEN: print("Warning: HF_TOKEN not found in environment variables. Submissions will not work.") api = HfApi(token=HF_TOKEN) DEFAULT_PARAMS = { "text":{ "dataset_name": "QuotaClimat/frugalaichallenge-text-train", "test_size": 0.2, # must be between 0 and 1 "test_seed": 42, # must be non-negative }, "image":{ "dataset_name": "pyronear/pyro-sdis", "test_size": 0.2, # must be between 0 and 1 "test_seed": 42, # must be non-negative }, "audio":{ "dataset_name": "rfcx/frugalai", "test_size": 0.2, # must be between 0 and 1 "test_seed": 42, # must be non-negative } } def evaluate_model(task: str, space_url: str): """ Evaluate a model through its API endpoint """ # username = space_url.split("/")[0] if "localhost" in space_url: api_url = f"{space_url}/{task}" else: api_url = f"https://{space_url.lower().replace('/', '-')}.hf.space/{task}" # Check if the space exists, will raise an error if it doesn't api.space_info(repo_id=space_url) try: # Make API call to the space params = DEFAULT_PARAMS[task] response = requests.post(api_url, json=params) if response.status_code != 200: return None, None, None, gr.Warning(f"API call failed with status {response.status_code}") results = response.json() # Check for required keys based on task base_required_keys = { "username", "space_url", "submission_timestamp", "model_description", "energy_consumed_wh", "emissions_gco2eq", "emissions_data", "api_route", "dataset_config" } # Add task-specific accuracy keys if task == "image": accuracy_keys = {"classification_accuracy", "mean_iou"} else: # text and audio accuracy_keys = {"accuracy"} required_keys = base_required_keys | accuracy_keys missing_keys = required_keys - set(results.keys()) if missing_keys: return None, None, None, gr.Warning(f"API response missing required keys: {', '.join(missing_keys)}") # Return appropriate accuracy metric based on task if task == "image": accuracy = results["classification_accuracy"] # For display in UI else: accuracy = results["accuracy"] return ( accuracy, results["emissions_gco2eq"], results["energy_consumed_wh"], results ) except Exception as e: return None, None, None, gr.Warning(str(e)) def submit_results(task: str, results_json): if not results_json: return gr.Warning("No results to submit") if not HF_TOKEN: return gr.Warning("HF_TOKEN not found. Please set up your Hugging Face token.") try: results_str = json.dumps(results_json) # Create a temporary file with the results with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.json') as f: f.write(results_str) temp_path = f.name # Upload to the appropriate dataset based on task api = HfApi(token=HF_TOKEN) path_in_repo = f"submissions/{results_json['username']}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" dataset_mapping = { "text": "frugal-ai-challenge/public-leaderboard-text", "image": "frugal-ai-challenge/public-leaderboard-image", "audio": "frugal-ai-challenge/public-leaderboard-audio" } api.upload_file( path_or_fileobj=temp_path, path_in_repo=path_in_repo, repo_id=dataset_mapping[task], repo_type="dataset", token=HF_TOKEN ) # Clean up os.unlink(temp_path) return gr.Info("Results submitted successfully to the leaderboard! 🎉") except Exception as e: return gr.Warning(f"Error submitting results: {str(e)}") # Create the demo interface with gr.Blocks() as demo: gr.Image("./logo.png", show_label=False, container=False) gr.Markdown(""" # Frugal AI Challenge - Submission Portal Submit your model results for any of the three tasks: Text, Image, or Audio classification. """) with gr.Tabs(): with gr.Tab("Instructions"): gr.Markdown(""" To submit your results in one of the three tasks, please follow the steps below: ## Prepare your model submission 1. Duplicate the template of the submission API by duplicating this space https://huggingface.co/spaces/frugal-ai-challenge/submission-template on your own Hugging Face account. 2. In ``tasks/text.py``, ``tasks/image.py``, or ``tasks/audio.py``, modify the ``evaluate_model`` function to replace the baseline by your model loading and inference within the inference pass where the energy consumption and emissions are tracked. 3. Eventually complete the requirements and/or any necessaries dependencies in your space. 4. Write down your model card in the ``README.md`` file. 5. Deploy your space (FastAPI) and verify that it works. 6. (Optional) You can change the Space hardware to use any GPU directly on Hugging Face. ## Submit your model to the leaderboard in the ``Model Submission`` tab When your API is deployed : 1. Select the task you want to submit your model to 2. Enter the Space URL of your API 3. (Optional) Precise the API route (default is ``/text``, ``/image``, or ``/audio``) 4. Step 1 - Evaluate model: Click on the button to evaluate your model. This will run you model on your API, computes the accuracy on the test set (20% of the train set), and track the energy consumption and emissions. 5. Step 2 - Submit to leaderboard: Click on the button to submit your results to the leaderboard. This will upload the results to the leaderboard dataset and update the leaderboard. 6. You can see the leaderboards at - Text - https://huggingface.co/datasets/frugal-ai-challenge/public-leaderboard-text - Image - https://huggingface.co/datasets/frugal-ai-challenge/public-leaderboard-image - Audio - https://huggingface.co/datasets/frugal-ai-challenge/public-leaderboard-audio ## About > You can find more information about the Frugal AI Challenge 2025 on the [Frugal AI Challenge website](https://frugal-ai-challenge.org/). > Or directly on the organization page on Hugging Face: [Frugal AI Challenge](https://huggingface.co/frugal-ai-challenge) This portal is a submission portal for the Frugal AI Challenge 2025. It is a simple interface to evaluate and submit your model to the leaderboard. The challenge is organized by Hugging Face, Data For Good, and the French Ministry of Environment. The goal of the Frugal AI Challenge is to encourage both academic and industry actors to keep efficiency in mind when deploying AI models. By tracking both energy consumption and performance for different AI tasks, we can incentivize frugality in AI deployment while also addressing real-world challenges. """) # Text Classification Task with gr.Tab("📜 Text Classification"): with gr.Row(): text_space_url = gr.Textbox( label="Space URL", placeholder="username/your-space", lines=1 ) text_route = gr.Textbox( label="API route (Advanced)", value="/text", lines=1 ) with gr.Row(): with gr.Column(scale=1): text_evaluate_btn = gr.Button("1. Evaluate model", variant="secondary") with gr.Column(scale=1): text_submit_btn = gr.Button("2. Submit to leaderboard", variant="primary") with gr.Row(): text_accuracy = gr.Number(label="Accuracy", precision=4) text_energy = gr.Number(label="Energy Consumed (Wh)", precision=12) text_emissions = gr.Number(label="Emissions (gCO2eq)", precision=12) with gr.Row(): text_results_json = gr.JSON(label="Detailed Results", visible=True) # Image Classification Task with gr.Tab("🎥 Image Classification"): with gr.Row(): image_space_url = gr.Textbox( label="Space URL", placeholder="username/your-space", lines=1 ) image_route = gr.Textbox( label="API route (Advanced)", value="/image", lines=1 ) with gr.Row(): with gr.Column(scale=1): image_evaluate_btn = gr.Button("1. Evaluate model", variant="secondary") with gr.Column(scale=1): image_submit_btn = gr.Button("2. Submit to leaderboard", variant="primary") with gr.Row(): image_accuracy = gr.Number(label="Accuracy", precision=4) image_energy = gr.Number(label="Energy Consumed (Wh)", precision=12) image_emissions = gr.Number(label="Emissions (gCO2eq)", precision=12) with gr.Row(): image_results_json = gr.JSON(label="Detailed Results", visible=True) # Audio Classification Task with gr.Tab("🔊 Audio Classification"): with gr.Row(): audio_space_url = gr.Textbox( label="Space URL", placeholder="username/your-space", lines=1 ) audio_route = gr.Textbox( label="API route (Advanced)", value="/audio", lines=1 ) with gr.Row(): with gr.Column(scale=1): audio_evaluate_btn = gr.Button("1. Evaluate model", variant="secondary") with gr.Column(scale=1): audio_submit_btn = gr.Button("2. Submit to leaderboard", variant="primary") with gr.Row(): audio_accuracy = gr.Number(label="Accuracy", precision=4) audio_energy = gr.Number(label="Energy Consumed (Wh)", precision=12) audio_emissions = gr.Number(label="Emissions (gCO2eq)", precision=12) with gr.Row(): audio_results_json = gr.JSON(label="Detailed Results", visible=True) # Set up event handlers text_evaluate_btn.click( lambda url, route: evaluate_model(route.strip("/"), url), inputs=[text_space_url, text_route], outputs=[text_accuracy, text_emissions, text_energy, text_results_json] ) image_evaluate_btn.click( lambda url, route: evaluate_model(route.strip("/"), url), inputs=[image_space_url, image_route], outputs=[image_accuracy, image_emissions, image_energy, image_results_json] ) audio_evaluate_btn.click( lambda url, route: evaluate_model(route.strip("/"), url), inputs=[audio_space_url, audio_route], outputs=[audio_accuracy, audio_emissions, audio_energy, audio_results_json] ) text_submit_btn.click( lambda results: submit_results("text", results), inputs=[text_results_json], outputs=None ) image_submit_btn.click( lambda results: submit_results("image", results), inputs=[image_results_json], outputs=None ) audio_submit_btn.click( lambda results: submit_results("audio", results), inputs=[audio_results_json], outputs=None ) if __name__ == "__main__": demo.launch()