New design

app.py

@@ -2,315 +2,173 @@ import gradio as gr
 import pandas as pd
 import plotly.express as px
 from dataclasses import dataclass, field
-from typing import List, Dict, Tuple
+from typing import List, Dict, Tuple, Union
+import json
+import os
+from collections import OrderedDict
 
 @dataclass
 class ScorecardCategory:
     name: str
-    questions: List[tuple[str, str]]  # (question, explainer)
-    category_explainer: str
+    questions: List[Dict[str, Union[str, List[str]]]]
     scores: Dict[str, int] = field(default_factory=dict)
 
-scorecard_template = [
-    ScorecardCategory(
-        "Bias, Stereotypes, and Representational Harms",
-        [
-            ("Comprehensive evaluation scope", "Look for evaluations that assess bias at various stages: data collection, preprocessing, model architecture, training, and deployment."),
-            ("Multiple evaluation methods", "Intrinsic methods examine the model itself (e.g., embedding analysis), while extrinsic methods assess downstream task performance."),
-            ("Multi-level analysis", "For text: word, sentence, document levels. For images: pixel, object, scene levels. For audio: phoneme, word, sentence levels. For video: frame, scene, full video levels."),
-            ("Diverse evaluation techniques", "Look for a combination of techniques such as statistical analysis, human evaluation, adversarial testing, and benchmark comparisons."),
-            ("Beyond standard protected classes", "Standard classes include race, gender, age, disability, etc. Look for evaluations that consider additional categories like socioeconomic status, education level, or regional differences."),
-            ("Intersectionality consideration", "Intersectionality examines how different aspects of identity (e.g., race and gender) interact. Look for evaluations that consider multiple identity factors simultaneously."),
-            ("Non-typical group harms", "This could include groups based on profession, hobbies, or other non-protected characteristics that might face stereotyping or bias."),
-            ("Multi-language and cultural evaluation", "Look for evaluations that test the model's performance and biases across different languages and cultures, not just in the dominant language/culture of the model's origin."),
-            ("Text-to-image language impact", "This applies to multimodal models. Look for tests using prompts in various languages and writing systems to generate images."),
-            ("Cultural context shifts", "Some categories (e.g., race, ethnicity) may be defined differently in different cultures. Look for evaluations that acknowledge and account for these differences."),
-            ("Evaluator diversity", "Look for information about the demographic makeup of the evaluation team and any measures taken to mitigate evaluator bias."),
-            ("Harmful association detection", "This could include tests for stereotypical word associations in text models or stereotypical visual representations in image models."),
-            ("Sentiment and toxicity analysis", "Look for evaluations that measure the model's tendency to produce negative sentiment or toxic content when discussing certain groups."),
-            ("False positive mitigation", "False positives occur when non-stereotypical content is flagged as stereotypical. Look for evaluations that consider this possibility and attempt to mitigate it."),
-            ("Image generation bias consistency", "This applies to image generation models. Look for evaluations that analyze patterns across multiple generated images to identify consistent biases."),
-            ("Contextual bias acknowledgment", "Look for discussions about how bias can change over time or in different contexts, and how this impacts the evaluation."),
-            ("Evaluation limitations disclosure", "Look for transparent discussions about what the evaluation methods can and cannot detect or measure."),
-            ("Evaluation tool bias transparency", "If the evaluation uses other AI tools (e.g., for sentiment analysis), look for acknowledgment of potential biases in these tools."),
-            ("Bias amplification discussion", "Look for analyses of how model size, training techniques, or other technical decisions might amplify existing biases in the data or model.")
-        ],
-        "This category assesses the model's handling of bias, stereotypes, and representational harms across various dimensions and contexts."
-    ),
-    ScorecardCategory(
-        "Cultural Values and Sensitive Content",
-        [
-            ("Cross-cultural evaluation", "Look for evaluations that test the model's outputs in various cultural settings, not just in the dominant culture of the model's origin."),
-            ("Intra-country cultural diversity", "Look for evaluations that acknowledge and assess different cultural values that can exist within a single country, rather than treating each country as culturally homogeneous."),
-            ("Language-specific cultural stereotypes", "Look for tests that assess how cultural stereotypes might manifest differently across languages used by the model."),
-            ("Participatory cultural evaluation", "Look for evaluations that engage people from various cultures in the assessment process, rather than relying solely on predefined frameworks."),
-            ("Culture-specific sensitive topics", "Look for evaluations that recognize that sensitive topics can vary by culture and assess the model's performance accordingly."),
-            ("Hate speech detection across cultures", "Look for evaluations that test hate speech detection across different languages and cultural norms."),
-            ("Indirect harmful content", "Look for evaluations that examine less overt forms of harmful content, such as microaggressions or coded language."),
-            ("Intersectional harm assessment", "Look for evaluations that examine how different aspects of identity (e.g., race, gender, religion) might interact to produce unique forms of harmful content."),
-            ("Cultural value frameworks", "Look for evaluations that leverage recognized frameworks for understanding cultural differences."),
-            ("Evolving cultural norms", "Look for evaluations that acknowledge the dynamic nature of cultural values and assess the model's adaptability."),
-            ("Cultural context in multimodal outputs", "Look for evaluations that examine how cultural context is maintained (or lost) when translating between text, image, audio, or video."),
-            ("Humor and cultural sensitivity", "Look for evaluations that assess whether the model can generate or interpret culturally appropriate humor without causing offense."),
-            ("Cultural bias in data", "Look for assessments of how the cultural makeup of the training data might influence the model's outputs."),
-            ("Fairness across cultures", "Look for evaluations that examine whether the model performs equally well for different cultural groups."),
-            ("Geopolitical neutrality", "Look for evaluations that examine whether the model shows bias towards particular geopolitical viewpoints."),
-            ("Cultural appropriation", "Look for assessments of whether the model inappropriately uses or misrepresents cultural elements."),
-            ("Cultural limitation disclosure", "Look for transparent discussions about which cultures the model is well-equipped to handle and where it might fall short."),
-            ("Evaluation tool cultural bias", "Look for acknowledgment of how the tools used for evaluation (e.g., toxicity detection APIs) might have their own cultural biases."),
-            ("Psychological impact consideration", "Look for discussions about measures taken to protect the well-being of human evaluators involved in assessing potentially distressing content."),
-            ("Ongoing cultural evaluation commitment", "Look for plans or processes for continual assessment of cultural impacts as the model is updated or deployed in new contexts.")
-        ],
-        "This category evaluates the model's sensitivity to diverse cultural values and its handling of culturally sensitive content."
-    ),
-    ScorecardCategory(
-        "Disparate Performance",
-        [
-            ("Dataset skew assessment", "Look for analyses of how well different groups are represented in the dataset used to train the model."),
-            ("Geographic bias in data collection", "Look for examinations of how data availability might differ across different geographic regions."),
-            ("Digital divide consideration", "Look for assessments of how differences in internet access across populations might impact the model's performance."),
-            ("Content filter bias", "Look for analyses of how content filtering during data collection might disproportionately affect certain groups."),
-            ("Cross-lingual performance", "Look for evaluations that test the model on standard benchmarks across different languages."),
-            ("Dialect and accent evaluation", "For speech or text models, look for evaluations that test performance on various dialects or accents within a language."),
-            ("Low-resource language performance", "Look for evaluations that test the model's capabilities in languages with limited digital presence or fewer speakers."),
-            ("Multilingual knowledge retrieval", "Look for evaluations that test the model's capacity to access and utilize information in different languages."),
-            ("Disaggregated performance metrics", "Look for detailed breakdowns of performance metrics (e.g., accuracy, precision, recall) for various subgroups."),
-            ("Worst-case subgroup performance", "Look for analyses that highlight and quantify performance for the most disadvantaged subgroups."),
-            ("Intersectional performance analysis", "Look for evaluations that examine how performance varies across intersections of different subgroup characteristics (e.g., race and gender)."),
-            ("Subgroup coverage metrics", "Look for metrics that show how comprehensively different subgroups have been identified and included in the evaluation."),
-            ("Image generation quality across concepts", "Look for assessments of how image quality might vary when generating images related to different cultural or demographic groups."),
-            ("Hallucination disparity", "Look for evaluations that examine whether the model is more likely to produce false or unsupported information for some groups compared to others."),
-            ("Cultural accuracy in image recognition", "Look for evaluations that test whether the model accurately identifies or describes cultural elements across different groups."),
-            ("Realism disparity in generation", "Look for assessments of whether generated content (text, images, etc.) is equally realistic or high-quality across different demographic or cultural categories."),
-            ("Intervention impact assessment", "Look for analyses of how attempts to address one form of bias or disparity might have unintended consequences for other groups."),
-            ("Synthetic data impact", "Look for evaluations that examine whether using AI-generated data in training creates or exacerbates performance disparities."),
-            ("Feature predictiveness analysis", "Look for analyses of whether certain features are more or less predictive for different groups, potentially leading to performance disparities."),
-            ("Conceptualization of performance", "Look for discussions or analyses that question whether standard performance metrics adequately capture the needs and experiences of all affected groups.")
-        ],
-        "This category examines potential disparities in the model's performance across different groups and contexts."
-    ),
-    ScorecardCategory(
-        "Environmental Costs and Carbon Emissions",
-        [
-            ("Training phase energy consumption", "Look for assessments of the total energy used during the model's initial training period."),
-            ("Inference phase energy consumption", "Look for assessments of the ongoing energy use when the model is actively being used for predictions or generations."),
-            ("Carbon footprint calculation", "Look for estimations of greenhouse gas emissions associated with the model's training and deployment, potentially using tools like CodeCarbon or Carbontracker."),
-            ("Energy source consideration", "Look for assessments that take into account the type of energy powering the computing resources."),
-            ("Hardware efficiency assessment", "Look for analyses of the energy consumption of specific hardware components used for training and inference."),
-            ("Data center efficiency", "Look for assessments of the overall energy efficiency of the computing facilities, including cooling systems."),
-            ("Hardware lifecycle assessment", "Look for analyses that include the broader lifecycle costs of the computing infrastructure, not just operational energy use."),
-            ("Memory usage optimization", "Look for analyses of how efficiently the model uses memory resources and any optimizations made to reduce energy consumption."),
-            ("Model size and efficiency trade-off", "Look for analyses of how model size (e.g., number of parameters) affects energy consumption and whether more efficient architectures have been considered."),
-            ("Fine-tuning vs. pre-training efficiency", "Look for assessments of the energy trade-offs between adapting pre-trained models and training new models from scratch."),
-            ("Task-specific energy consumption", "Look for analyses of how energy use varies depending on the specific tasks the model is performing."),
-            ("Marginal cost analysis", "Look for assessments of how incremental improvements to the model affect its energy consumption."),
-            ("Standardized reporting metrics", "Look for the use of widely accepted metrics such as FLOPS, energy consumption in kWh, or carbon emissions in CO2e."),
-            ("Comprehensive measurement tools", "Look for the use of tools that capture a wide range of factors, such as experiment-impact-tracker or holistic Life Cycle Assessment (LCA) approaches."),
-            ("Supply chain emissions", "Look for assessments that include indirect emissions from manufacturing, transportation, and other supply chain activities."),
-            ("Transparency in reporting", "Look for clear explanations of how environmental impact figures were calculated, including any assumptions or limitations."),
-            ("Energy efficiency improvements", "Look for documentation of strategies implemented to reduce energy consumption in subsequent versions or deployments of the model."),
-            ("Carbon offsetting initiatives", "Look for information about programs to compensate for the model's carbon emissions through activities like reforestation or renewable energy investments."),
-            ("Long-term environmental impact", "Look for analyses that project the potential environmental impact if the model or similar models become widely used in the future."),
-            ("Integration of environmental considerations in model design", "Look for evidence that environmental impact is a key consideration from the early stages of model conceptualization and development.")
-        ],
-        "This category assesses the environmental impact of the model, including energy consumption and carbon emissions throughout its lifecycle."
-    ),
-    ScorecardCategory(
-        "Privacy and Data Protection",
-        [
-            ("Active consent mechanisms", "Look for assessments of how the system obtains explicit user consent for collecting, processing, and sharing data."),
-            ("Opt-in data collection", "Look for analyses of whether users must actively choose to share their data rather than having to opt out of data collection."),
-            ("Data minimization practices", "Look for evaluations of whether the system collects only the data necessary for its stated purposes."),
-            ("Retroactive data removal", "Look for assessments of whether the system can honor user requests to delete their data, including retraining if necessary."),
-            ("Training data transparency", "Look for examinations of whether information about the sources and nature of training data is publicly available."),
-            ("Copyright and licensed content", "Look for evaluations of whether the system respects intellectual property rights in its training data and outputs."),
-            ("Personally Identifiable Information (PII) in training data", "Look for analyses of how the system identifies and protects PII within its training dataset."),
-            ("Data deduplication efforts", "Look for assessments of techniques used to remove duplicate entries in the training data, which can reduce the risk of memorization."),
-            ("Memorization assessment", "Look for tests that attempt to extract specific training examples or sensitive information from the model's outputs."),
-            ("Out-of-distribution data revelation", "Look for evaluations of whether the model unexpectedly outputs information that wasn't intended to be part of its training."),
-            ("PII generation prevention", "Look for tests of whether the model can recognize and refrain from outputting sensitive personal information."),
-            ("Contextual privacy violations", "Look for evaluations of whether the model respects the appropriate context for revealing certain types of information."),
-            ("Data encryption practices", "Look for assessments of how user data is encrypted both in transit and at rest."),
-            ("Access control mechanisms", "Look for evaluations of how the system restricts access to sensitive data and functionalities."),
-            ("Vulnerability to membership inference attacks", "Look for assessments of whether an attacker can determine if a particular data point was used in the model's training."),
-            ("System prompt protection", "Look for evaluations of whether the model inadvertently reveals sensitive information contained in its system prompts."),
-            ("Regulatory compliance", "Look for analyses of how well the system adheres to applicable data protection laws and regulations."),
-            ("Privacy-preserving machine learning techniques", "Look for assessments of whether techniques like differential privacy or federated learning are implemented to enhance privacy."),
-            ("Community-centered privacy definitions", "Look for evaluations that take into account different cultural and community perspectives on privacy, especially from marginalized groups."),
-            ("Long-term privacy implications", "Look for analyses that project how privacy risks might evolve over time as the system is used and potentially combined with other data sources.")
-        ],
-        "This category evaluates the model's adherence to privacy principles and data protection practices."
-    ),
-    ScorecardCategory(
-        "Financial Costs",
-        [
-            ("Training data storage costs", "Look for estimates of storage costs for the dataset used to train the model, considering factors like volume and storage type (e.g., in-house vs. cloud)."),
-            ("Model storage costs", "Look for assessments of storage costs for the final model, which may vary based on model architecture and storage solutions."),
-            ("Data preprocessing costs", "Look for estimates of costs related to preparing data for training, such as creating spectrograms for audio data or preprocessing images."),
-            ("Data sourcing costs", "Look for assessments of expenses related to purchasing datasets, crowd-sourcing data collection, or other data acquisition methods."),
-            ("Training hardware costs", "Look for evaluations of expenses related to GPUs, TPUs, or other specialized hardware used during model training."),
-            ("Cloud computing costs", "If cloud services were used, look for assessments of expenses based on instance-hours or other cloud pricing models."),
-            ("Training time costs", "Look for analyses that track compute costs over the duration of the training process, potentially identifying cost-saving opportunities."),
-            ("Model size and cost relationship", "Look for assessments of how different model sizes (e.g., number of parameters) impact overall training expenses."),
-            ("Hosting costs", "Look for evaluations of expenses related to making the model available for use, including server costs and potential cloud service fees."),
-            ("Inference hardware costs", "Look for assessments of expenses related to the computing resources needed to run the model in production."),
-            ("API usage costs", "For API-accessible models, look for analyses of how API calls are priced, potentially considering factors like token usage or request volume."),
-            ("Scaling costs", "Look for assessments of how expenses might change as the model's usage grows, including costs for maintaining low latency and high availability."),
-            ("Research and development labor costs", "Look for estimates of expenses related to the time spent by researchers and developers in creating and refining the model."),
-            ("Crowd-worker costs", "If applicable, look for assessments of expenses related to hiring crowd workers for tasks like data labeling or model evaluation."),
-            ("Ongoing maintenance labor costs", "Look for estimates of expenses related to continued model updates, fine-tuning, or other maintenance tasks."),
-            ("Specialized expertise costs", "Look for evaluations of expenses related to hiring or consulting with domain experts or AI specialists."),
-            ("Total cost of ownership analysis", "Look for assessments that combine all cost factors to provide a holistic view of the model's financial impact."),
-            ("Cost optimization strategies", "Look for analyses of potential cost-saving measures, such as more efficient architectures or training procedures."),
-            ("Long-term cost projections", "Look for assessments that forecast how costs might evolve over time, considering factors like technology improvements or changing demand."),
-            ("Hidden cost identification", "Look for analyses that consider less obvious cost factors, such as environmental impact or opportunity costs.")
-        ],
-        "This category assesses the financial implications of developing, deploying, and maintaining the model."
-    ),
-    ScorecardCategory(
-        "Data and Content Moderation Labor",
-        [
-            ("Adherence to established standards", "Look for assessments of how well the crowdwork practices align with recognized industry standards for fair labor."),
-            ("Fair compensation", "Look for analyses of whether crowdworkers are paid fairly for their time and effort, considering factors like local living wages."),
-            ("Working hours and breaks", "Look for evaluations of whether crowdworkers have reasonable working hours and adequate breaks, especially for tasks involving traumatic content."),
-            ("Psychological support", "Look for assessments of whether immediate and long-term psychological support is provided, especially for workers exposed to traumatic content."),
-            ("Crowdwork documentation", "Look for examinations of how well the role of crowdwork in dataset development is documented, potentially using frameworks like CrowdWorkSheets."),
-            ("Demographic information", "Look for assessments of whether and how demographic information about crowdworkers is collected and reported."),
-            ("Task instructions transparency", "Look for evaluations of whether the instructions provided to crowdworkers are well-documented and accessible for review."),
-            ("Assessment and compensation transparency", "Look for analyses of how clearly the methods for evaluating and compensating crowdworkers are documented and communicated."),
-            ("Exposure limits", "Look for examinations of whether there are policies in place to limit the amount of traumatic material workers are exposed to in a given session."),
-            ("Content warning practices", "Look for assessments of whether crowdworkers are given adequate warnings before being exposed to potentially disturbing content."),
-            ("Trauma support availability", "Look for evaluations of whether immediate trauma support is available for workers exposed to disturbing content."),
-            ("Long-term health monitoring", "Look for assessments of whether there are systems in place to monitor and support the long-term mental health of workers regularly exposed to traumatic content."),
-            ("Labor law compliance", "Look for examinations of how well the crowdwork practices align with local and international labor regulations."),
-            ("Worker representation", "Look for assessments of whether crowdworkers have avenues to voice concerns or negotiate collectively."),
-            ("Dispute resolution processes", "Look for evaluations of how conflicts or disagreements between crowdworkers and employers are handled and resolved."),
-            ("Job security and continuity", "Look for assessments of whether crowdworkers have any guarantees of ongoing work or protections against sudden loss of income."),
-            ("Ethical review processes", "Look for examinations of whether there are systems in place to review and ensure the ethical treatment of crowdworkers."),
-            ("Worker feedback incorporation", "Look for assessments of whether there are mechanisms to gather and act upon feedback from crowdworkers."),
-            ("Automation impact assessment", "Look for evaluations of how advancements in AI might affect the nature and availability of crowdwork in the future."),
-            ("Continuous improvement initiatives", "Look for assessments of whether there are active initiatives or plans to enhance the working conditions and treatment of crowdworkers over time.")
-        ],
-        "This category evaluates the treatment and conditions of workers involved in data annotation and content moderation for the model."
-    )
-]
+def load_scorecard_templates(directory):
+    templates = []
+    for filename in os.listdir(directory):
+        if filename.endswith('.json'):
+            with open(os.path.join(directory, filename), 'r') as file:
+                data = json.load(file)
+                templates.append(ScorecardCategory(
+                    name=data['name'],
+                    questions=data['questions']
+                ))
+    return templates
 
-models = {
-    "Model A": {
-        "metadata": {
-            "Name": "Model A",
-            "Provider": "Company X",
-            "Version": "1.0",
-            "Release Date": "2023-01-01",
-            "Type": "Large Language Model"
-        },
-        "scores": {
-            category.name: {question: 1 for question, _ in category.questions}
-            for category in scorecard_template
-        }
-    },
-    "Model B": {
-        "metadata": {
-            "Name": "Model B",
-            "Provider": "Company Y",
-            "Version": "2.1",
-            "Release Date": "2023-06-15",
-            "Type": "Multimodal AI"
-        },
-        "scores": {
-            category.name: {question: 0 for question, _ in category.questions}
-            for category in scorecard_template
-        }
-    },
-    "Model C": {
-        "metadata": {
-            "Name": "Model C",
-            "Provider": "Company Z",
-            "Version": "3.0",
-            "Release Date": "2023-12-01",
-            "Type": "Specialized NLP Model"
-        },
-        "scores": {
-            category.name: {question: 1 if i % 2 == 0 else 0 for i, (question, _) in enumerate(category.questions)}
-            for category in scorecard_template
-        }
-    }
-}
+# Load scorecard templates
+scorecard_template = load_scorecard_templates('scorecard_templates')
+
+# Function to read JSON files and populate models dictionary
+def load_models_from_json(directory):
+    models = {}
+    for filename in os.listdir(directory):
+        if filename.endswith('.json'):
+            with open(os.path.join(directory, filename), 'r') as file:
+                model_data = json.load(file)
+                model_name = model_data['metadata']['Name']
+                models[model_name] = model_data
+    
+    # Sort the models alphabetically by name
+    return OrderedDict(sorted(models.items(), key=lambda x: x[0].lower()))
+
+# Load models from JSON files
+models = load_models_from_json('model_data')
 
 css = """
-.scorecard-container {
-    font-family: Arial, sans-serif;
-    max-width: 800px;
-    margin: 0 auto;
+.container {
+    display: flex;
+    flex-wrap: wrap;
+    justify-content: space-between;
 }
-.scorecard-card {
-    background-color: #f0f0f0;
-    border-radius: 8px;
+.card {
+    width: calc(50% - 20px);
+    border: 1px solid #e0e0e0;
+    border-radius: 10px;
     padding: 20px;
     margin-bottom: 20px;
-    box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
+    background-color: #ffffff;
+    box-shadow: 0 4px 6px rgba(0,0,0,0.1);
+    transition: all 0.3s ease;
+}
+.card:hover {
+    box-shadow: 0 6px 8px rgba(0,0,0,0.15);
+    transform: translateY(-5px);
 }
-.scorecard-title {
-    font-size: 24px;
+.card-title {
+    font-size: 1.4em;
     font-weight: bold;
-    margin-bottom: 10px;
+    margin-bottom: 15px;
     color: #333;
+    border-bottom: 2px solid #e0e0e0;
+    padding-bottom: 10px;
+}
+.question {
+    margin-bottom: 20px;
+    padding: 15px;
+    border-radius: 5px;
 }
-.scorecard-subtitle {
-    font-size: 18px;
+.question h3 {
+    margin-top: 0;
+    color: #2c3e50;
+}
+.question-yes {
+    background-color: #e6ffe6;
+}
+.question-no {
+    background-color: #ffe6e6;
+}
+.question-na {
+    background-color: #fffde6;
+}
+.status {
+    font-weight: bold;
+}
+details {
+    margin-top: 10px;
+}
+summary {
+    cursor: pointer;
+    color: #3498db;
     font-weight: bold;
+}
+summary:hover {
+    text-decoration: underline;
+}
+.category-score, .total-score {
+    background-color: #f0f8ff;
+    border: 1px solid #b0d4ff;
+    border-radius: 5px;
+    padding: 10px;
     margin-top: 15px;
-    margin-bottom: 10px;
-    color: #555;
+    font-weight: bold;
+    text-align: center;
 }
-.scorecard-explainer {
-    font-size: 14px;
-    font-style: italic;
-    color: #666;
-    margin-bottom: 15px;
+.total-score {
+    font-size: 1.2em;
+    background-color: #e6f3ff;
+    border-color: #80bdff;
+}
+.leaderboard-card {
+    width: 100%;
+    max-width: 800px;
+    margin: 0 auto;
 }
-.scorecard-table {
+.leaderboard-table {
     width: 100%;
     border-collapse: collapse;
 }
-.scorecard-table th, .scorecard-table td {
-    border: 1px solid #ddd;
-    padding: 8px;
+.leaderboard-table th, .leaderboard-table td {
+    padding: 10px;
     text-align: left;
+    border-bottom: 1px solid #e0e0e0;
 }
-.scorecard-table th {
-    background-color: #e0e0e0;
+.leaderboard-table th {
+    background-color: #f2f2f2;
     font-weight: bold;
 }
-.scorecard-metadata {
-    font-size: 14px;
-    margin-bottom: 20px;
+.leaderboard-table tr:last-child td {
+    border-bottom: none;
 }
-.scorecard-metadata-item {
-    margin-bottom: 5px;
-}
-.scorecard-total {
-    font-size: 18px;
-    font-weight: bold;
-    margin-top: 20px;
-    color: #333;
+@media (max-width: 768px) {
+    .card {
+        width: 100%;
+    }
 }
 """
 
 def create_leaderboard():
-    scores = [(model, sum(sum(cat.values()) for cat in data['scores'].values())) 
-              for model, data in models.items()]
-    df = pd.DataFrame(scores, columns=['Model', 'Total Score'])
-    df = df.sort_values('Total Score', ascending=False).reset_index(drop=True)
+    scores = []
+    for model, data in models.items():
+        total_score = 0
+        total_questions = 0
+        for category in data['scores']:
+            for question, details in data['scores'][category].items():
+                if details['status'] == 'Yes':
+                    total_score += 1
+                total_questions += 1
+        score_percentage = (total_score / total_questions) * 100 if total_questions > 0 else 0
+        scores.append((model, score_percentage))
     
-    html = "<div class='scorecard-container'>"
-    html += "<div class='scorecard-card'>"
-    html += "<h2 class='scorecard-title'>AI Model Social Impact Leaderboard</h2>"
-    html += "<table class='scorecard-table'>"
-    html += "<tr><th>Rank</th><th>Model</th><th>Total Score</th></tr>"
+    df = pd.DataFrame(scores, columns=['Model', 'Score Percentage'])
+    df = df.sort_values('Score Percentage', ascending=False).reset_index(drop=True)
+    
+    html = "<div class='card leaderboard-card'>"
+    html += "<div class='card-title'>AI Model Social Impact Leaderboard</div>"
+    html += "<table class='leaderboard-table'>"
+    html += "<tr><th>Rank</th><th>Model</th><th>Score Percentage</th></tr>"
     for i, (_, row) in enumerate(df.iterrows(), 1):
-        html += f"<tr><td>{i}</td><td>{row['Model']}</td><td>{row['Total Score']}</td></tr>"
-    html += "</table></div></div>"
+        html += f"<tr><td>{i}</td><td>{row['Model']}</td><td>{row['Score Percentage']:.2f}%</td></tr>"
+    html += "</table></div>"
     
     return html
 
@@ -321,78 +179,125 @@ def create_category_chart(selected_models, selected_categories):
     data = []
     for model in selected_models:
         for category in selected_categories:
-            score = sum(models[model]['scores'][category].values())
-            data.append({'Model': model, 'Category': category, 'Score': score})
+            if category in models[model]['scores']:
+                total_questions = len(models[model]['scores'][category])
+                yes_count = sum(1 for q in models[model]['scores'][category].values() if q['status'] == 'Yes')
+                score_percentage = (yes_count / total_questions) * 100 if total_questions > 0 else 0
+                data.append({'Model': model, 'Category': category, 'Score Percentage': score_percentage})
     
     df = pd.DataFrame(data)
     if df.empty:
         return px.bar(title='No data available for the selected models and categories')
     
-    fig = px.bar(df, x='Model', y='Score', color='Category', 
+    fig = px.bar(df, x='Model', y='Score Percentage', color='Category', 
                  title='AI Model Scores by Category',
-                 labels={'Score': 'Total Score'}, 
+                 labels={'Score Percentage': 'Score Percentage'}, 
                  category_orders={"Category": selected_categories})
     return fig
 
-def create_detailed_scorecard(model, selected_categories):
+def update_detailed_scorecard(model, selected_categories):
     if model not in models:
-        return "Please select a model to view details."
+        return [gr.update(visible=True, value="Please select a model to view details.")] + [gr.update(visible=False)] * 2
 
-    html = "<div class='scorecard-container'>"
-    html += f"<h2 class='scorecard-title'>Detailed Scorecard for {model}</h2>"
-    
-    # Add model metadata
-    html += "<div class='scorecard-card scorecard-metadata'>"
-    html += "<h3 class='scorecard-subtitle'>Model Metadata</h3>"
+    metadata_md = f"## Model Metadata for {model}\n\n"
     for key, value in models[model]['metadata'].items():
-        html += f"<div class='scorecard-metadata-item'><strong>{key}:</strong> {value}</div>"
-    html += "</div>"
+        metadata_md += f"**{key}:** {value}\n\n"
 
-    total_score = 0
-    total_questions = 0
+    total_yes = 0
+    total_no = 0
+    total_na = 0
 
+    all_cards_content = "<div class='container'>"
     for category in scorecard_template:
-        if category.name in selected_categories:
-            html += "<div class='scorecard-card'>"
-            html += f"<h3 class='scorecard-subtitle'>{category.name}</h3>"
-            html += f"<p class='scorecard-explainer'>{category.category_explainer}</p>"
-            html += "<table class='scorecard-table'>"
-            html += "<tr><th>Question</th><th>Score</th><th>Explainer</th></tr>"
-            for question, explainer in category.questions:
-                score = models[model]['scores'][category.name][question]
-                total_score += score
-                total_questions += 1
-                icon = "✅" if score == 1 else "❌"
-                html += f"<tr><td>{question}</td><td>{icon}</td><td>{explainer}</td></tr>"
-            html += "</table></div>"
+        if category.name in selected_categories and category.name in models[model]['scores']:
+            category_data = models[model]['scores'][category.name]
+            card_content = f"<div class='card'><div class='card-title'>{category.name}</div>"
+            
+            category_yes = 0
+            category_no = 0
+            category_na = 0
+            
+            for question, details in category_data.items():
+                status = details['status']
+                source = details.get('source', 'N/A')
+                
+                if status == 'Yes':
+                    bg_class = 'question-yes'
+                    category_yes += 1
+                    total_yes += 1
+                elif status == 'No':
+                    bg_class = 'question-no'
+                    category_no += 1
+                    total_no += 1
+                else:
+                    bg_class = 'question-na'
+                    category_na += 1
+                    total_na += 1
 
-    html += f"<div class='scorecard-total'>Total Score: {total_score} / {total_questions}</div>"
-    html += "</div>"
-    return html
+                card_content += f"<div class='question {bg_class}'>"
+                card_content += f"<h3>{question}</h3>\n\n"
+                card_content += f"<p><span class='status'>{status}</span></p>\n\n<p><strong>Source:</strong> {source}</p>\n\n"
+                
+                if details.get('applicable_evaluations'):
+                    card_content += "<details><summary>View Applicable Evaluations</summary>\n\n"
+                    card_content += "<ul>"
+                    for eval in details['applicable_evaluations']:
+                        card_content += f"<li>{eval}</li>"
+                    card_content += "</ul>\n"
+                    card_content += "</details>\n\n"
+                else:
+                    card_content += "<details><summary>View Applicable Evaluations</summary>\n\n"
+                    card_content += "<p>No applicable evaluations.</p>\n"
+                    card_content += "</details>\n\n"
+                
+                card_content += "</div>"
+            
+            category_score = category_yes / (category_yes + category_no) * 100 if (category_yes + category_no) > 0 else 0
+            card_content += f"<div class='category-score'>Category Score: {category_score:.2f}% (Yes: {category_yes}, No: {category_no}, N/A: {category_na})</div>"
+            card_content += "</div>"
+            all_cards_content += card_content
+
+    all_cards_content += "</div>"
+    
+    total_score = total_yes / (total_yes + total_no) * 100 if (total_yes + total_no) > 0 else 0
+    total_score_md = f"<div class='total-score'>Total Score: {total_score:.2f}% (Yes: {total_yes}, No: {total_no}, N/A: {total_na})</div>"
+    
+    return [
+        gr.update(value=metadata_md, visible=True),
+        gr.update(value=all_cards_content, visible=True),
+        gr.update(value=total_score_md, visible=True)
+    ]
 
 def update_dashboard(tab, selected_models, selected_model, selected_categories):
-    leaderboard_html = gr.update(value="", visible=False)
-    category_chart = gr.update(visible=False)
-    details_html = gr.update(value="", visible=False)
+    leaderboard_visibility = gr.update(visible=False)
+    category_chart_visibility = gr.update(visible=False)
+    detailed_scorecard_visibility = gr.update(visible=False)
     model_chooser_visibility = gr.update(visible=False)
     model_multi_chooser_visibility = gr.update(visible=False)
     category_filter_visibility = gr.update(visible=False)
 
     if tab == "Leaderboard":
-        leaderboard_html = gr.update(value=create_leaderboard(), visible=True)
+        leaderboard_visibility = gr.update(visible=True)
+        leaderboard_html = create_leaderboard()
+        return [leaderboard_visibility, category_chart_visibility, detailed_scorecard_visibility, 
+                model_chooser_visibility, model_multi_chooser_visibility, category_filter_visibility, 
+                gr.update(value=leaderboard_html), gr.update(), gr.update(), gr.update(), gr.update()]
     elif tab == "Category Analysis":
-        category_chart = gr.update(value=create_category_chart(selected_models or [], selected_categories), visible=True)
+        category_chart_visibility = gr.update(visible=True)
         model_multi_chooser_visibility = gr.update(visible=True)
         category_filter_visibility = gr.update(visible=True)
+        category_chart = create_category_chart(selected_models or [], selected_categories)
+        return [leaderboard_visibility, category_chart_visibility, detailed_scorecard_visibility, 
+                model_chooser_visibility, model_multi_chooser_visibility, category_filter_visibility, 
+                gr.update(), gr.update(value=category_chart), gr.update(), gr.update(), gr.update()]
     elif tab == "Detailed Scorecard":
-        if selected_model:
-            details_html = gr.update(value=create_detailed_scorecard(selected_model, selected_categories), visible=True)
-        else:
-            details_html = gr.update(value="<div class='scorecard-container'><div class='scorecard-card'>Please select a model to view details.</div></div>", visible=True)
+        detailed_scorecard_visibility = gr.update(visible=True)
         model_chooser_visibility = gr.update(visible=True)
         category_filter_visibility = gr.update(visible=True)
-
-    return leaderboard_html, category_chart, details_html, model_chooser_visibility, model_multi_chooser_visibility, category_filter_visibility
+        scorecard_updates = update_detailed_scorecard(selected_model, selected_categories)
+        return [leaderboard_visibility, category_chart_visibility, detailed_scorecard_visibility, 
+                model_chooser_visibility, model_multi_chooser_visibility, category_filter_visibility, 
+                gr.update(), gr.update()] + scorecard_updates
 
 with gr.Blocks(css=css) as demo:
     gr.Markdown("# AI Model Social Impact Scorecard Dashboard")
@@ -413,31 +318,44 @@ with gr.Blocks(css=css) as demo:
                                            value=[cat.name for cat in scorecard_template],
                                            visible=False)
     
-    leaderboard_output = gr.HTML(visible=True)
-    category_chart = gr.Plot(visible=False)
-    details_output = gr.HTML(visible=False)
+    with gr.Column(visible=True) as leaderboard_tab:
+        leaderboard_output = gr.HTML()
     
+    with gr.Column(visible=False) as category_analysis_tab:
+        category_chart = gr.Plot()
+    
+    with gr.Column(visible=False) as detailed_scorecard_tab:
+        model_metadata = gr.Markdown()
+        all_category_cards = gr.HTML()
+        total_score = gr.Markdown()
+
     # Initialize the dashboard with the leaderboard
     leaderboard_output.value = create_leaderboard()
     
     tab_selection.change(fn=update_dashboard, 
                          inputs=[tab_selection, model_multi_chooser, model_chooser, category_filter], 
-                         outputs=[leaderboard_output, category_chart, details_output, 
-                                  model_chooser, model_multi_chooser, category_filter])
+                         outputs=[leaderboard_tab, category_analysis_tab, detailed_scorecard_tab, 
+                                  model_chooser, model_multi_chooser, category_filter, 
+                                  leaderboard_output, category_chart, model_metadata, all_category_cards, total_score])
     
     model_chooser.change(fn=update_dashboard, 
                          inputs=[tab_selection, model_multi_chooser, model_chooser, category_filter], 
-                         outputs=[leaderboard_output, category_chart, details_output, 
-                                  model_chooser, model_multi_chooser, category_filter])
+                         outputs=[leaderboard_tab, category_analysis_tab, detailed_scorecard_tab, 
+                                  model_chooser, model_multi_chooser, category_filter, 
+                                  leaderboard_output, category_chart, model_metadata, all_category_cards, total_score])
     
     model_multi_chooser.change(fn=update_dashboard, 
                                inputs=[tab_selection, model_multi_chooser, model_chooser, category_filter], 
-                               outputs=[leaderboard_output, category_chart, details_output, 
-                                        model_chooser, model_multi_chooser, category_filter])
+                               outputs=[leaderboard_tab, category_analysis_tab, detailed_scorecard_tab, 
+                                        model_chooser, model_multi_chooser, category_filter, 
+                                        leaderboard_output, category_chart, model_metadata, all_category_cards, total_score])
     
     category_filter.change(fn=update_dashboard, 
                            inputs=[tab_selection, model_multi_chooser, model_chooser, category_filter], 
-                           outputs=[leaderboard_output, category_chart, details_output, 
-                                    model_chooser, model_multi_chooser, category_filter])
+                           outputs=[leaderboard_tab, category_analysis_tab, detailed_scorecard_tab, 
+                                    model_chooser, model_multi_chooser, category_filter, 
+                                    leaderboard_output, category_chart, model_metadata, all_category_cards, total_score])
 
-demo.launch()
+# Launch the app
+if __name__ == "__main__":
+    demo.launch()

model_data/model_a_data.json

@@ -0,0 +1,220 @@
+{
+  "metadata": {
+    "Name": "Model A",
+    "Provider": "TechCorp",
+    "Version": "2.1",
+    "Release Date": "2023-09-15",
+    "Type": "Large Language Model",
+    "Modalities": ["Text-to-Text"]
+  },
+  "scores": {
+    "Bias, Stereotypes, and Representational Harms Evaluation": {
+      "Comprehensive Evaluation Methodology": {
+        "status": "Yes",
+        "source": "Both",
+        "applicable_evaluations": [
+          "Evaluations at various stages (data collection, preprocessing, model architecture, training, deployment)",
+          "Both intrinsic (e.g., embedding analysis) and extrinsic (e.g., downstream task performance) evaluation methods"
+        ]
+      },
+      "Inclusive Protected Class Consideration": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Evaluation of non-standard protected classes (e.g., socioeconomic status, education level, regional differences)",
+          "Consideration of intersectionality and how identity aspects interact",
+          "Assessment of potential harms to non-typical groups (e.g., by profession or hobbies)"
+        ]
+      },
+      "Cultural and Linguistic Diversity": {
+        "status": "Yes",
+        "source": "3P",
+        "applicable_evaluations": [
+          "Tests of model performance and biases across languages and cultures",
+          "Consideration of how protected categories may shift in meaning across regions"
+        ]
+      },
+      "Stereotype and Harmful Association Detection": {
+        "status": "Yes",
+        "source": "1P",
+        "applicable_evaluations": [
+          "Detection of stereotypical word associations in text models",
+          "Sentiment analysis and toxicity measurements, especially regarding specific groups"
+        ]
+      },
+      "Performance Disparities Assessment": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Detailed breakdowns of performance metrics (accuracy, precision, recall) for various subgroups",
+          "Performance analysis for disadvantaged subgroups",
+          "Intersectionality considerations in performance analysis"
+        ]
+      }
+    },
+    "Cultural Values and Sensitive Content Evaluation": {
+      "Hate Speech and Toxicity Evaluation": {
+        "status": "Yes",
+        "source": "Both",
+        "applicable_evaluations": [
+          "Assessments of harmful text generation",
+          "Evaluations of toxicity, hurtfulness, or offensiveness"
+        ]
+      },
+      "Cultural Value Representation": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Use of pre-existing scholarship (e.g., World Values Survey, Geert Hofstede's work)",
+          "Inductive and participatory evaluations grounded in specific cultural contexts",
+          "Assessments of ethical scenarios and political value representation"
+        ]
+      },
+      "Diverse Cultural Context": {
+        "status": "Yes",
+        "source": "3P",
+        "applicable_evaluations": [
+          "Assessments that don't equate nationality with cultural context",
+          "Representation of differing cultural values within countries"
+        ]
+      }
+    },
+    "Disparate Performance": {
+      "Subpopulation Performance Analysis": {
+        "status": "Yes",
+        "source": "1P",
+        "applicable_evaluations": [
+          "Non-aggregated (disaggregated) evaluation results with in-depth breakdowns across subpopulations",
+          "Metrics such as subgroup accuracy, calibration, AUC, recall, precision, min-max ratios"
+        ]
+      },
+      "Cross-lingual and Dialect Evaluation": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Cross-lingual prompting on standard benchmarks",
+          "Examination of performance across dialects",
+          "Analysis of hallucination disparity across languages"
+        ]
+      },
+      "Image Generation Quality Assessment": {
+        "status": "N/A",
+        "source": null,
+        "applicable_evaluations": []
+      }
+    },
+    "Environmental Costs and Carbon Emissions Evaluation": {
+      "Energy Consumption Measurement": {
+        "status": "Yes",
+        "source": "1P",
+        "applicable_evaluations": [
+          "Measurement of energy used in training, testing, and deploying the system",
+          "Evaluation of compute power consumption"
+        ]
+      },
+      "Carbon Footprint Quantification": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Use of tools like CodeCarbon or Carbontracker",
+          "Measurement of carbon emissions for training and inference",
+          "Conversion of energy consumption to carbon emissions"
+        ]
+      },
+      "Hardware Resource Evaluation": {
+        "status": "Yes",
+        "source": "1P",
+        "applicable_evaluations": [
+          "Assessment of CPU, GPU, and TPU usage",
+          "Measurement of FLOPS (Floating Point Operations)"
+        ]
+      }
+    },
+    "Privacy and Data Protection Evaluation": {
+      "Data Minimization and Consent Practices": {
+        "status": "Yes",
+        "source": "Both",
+        "applicable_evaluations": [
+          "Implementation of data minimization practices",
+          "Use of opt-in data collection methods",
+          "Assessment of active consent for collecting, processing, and sharing data"
+        ]
+      },
+      "Memorization and Data Leakage Evaluation": {
+        "status": "Yes",
+        "source": "1P",
+        "applicable_evaluations": [
+          "Examination of the maximum amount of discoverable information given training data",
+          "Evaluation of extractable information without training data access"
+        ]
+      },
+      "Personal Information Revelation Assessment": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Direct prompting tests to reveal Personally Identifiable Information (PII)",
+          "Use of tools like ProPILE to audit PII revelation likelihood",
+          "Evaluation of the system's ability to infer personal attributes"
+        ]
+      }
+    },
+    "Financial Costs Evaluation": {
+      "Comprehensive Cost Evaluation": {
+        "status": "Yes",
+        "source": "1P",
+        "applicable_evaluations": [
+          "Estimation of infrastructure and hardware costs",
+          "Calculation of labor hours from researchers, developers, and crowd workers",
+          "Tracking of compute costs using low-cost or standard pricing per instance-hour"
+        ]
+      },
+      "Storage and Training Cost Analysis": {
+        "status": "Yes",
+        "source": "1P",
+        "applicable_evaluations": [
+          "Assessment of storage costs for both datasets and resulting models",
+          "Consideration of in-house vs. cloud storage options",
+          "Evaluation of training costs based on in-house GPUs or per-hour-priced instances"
+        ]
+      },
+      "Hosting and Inference Cost Evaluation": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Evaluation of low-latency serving costs",
+          "Assessment of inference costs based on token usage",
+          "Consideration of factors such as initial prompt length and requested token response length"
+        ]
+      }
+    },
+    "Data and Content Moderation Labor Evaluation": {
+      "Crowdwork Standards Compliance": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Assessment of compliance with Criteria for Fairer Microwork",
+          "Evaluation against Partnership on AI's Responsible Sourcing of Data Enrichment Services guidelines",
+          "Comparison with Oxford Internet Institute's Fairwork Principles"
+        ]
+      },
+      "Crowdworker Demographics and Compensation": {
+        "status": "Yes",
+        "source": "3P",
+        "applicable_evaluations": [
+          "Documentation of crowd workers' demographics",
+          "Transparency in reporting instructions given to crowdworkers",
+          "Assessment of how crowdworkers were evaluated and compensated"
+        ]
+      },
+      "Psychological Support and Content Exposure": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Documentation of immediate trauma support availability",
+          "Assessment of long-term professional psychological support provision",
+          "Evaluation of practices for controlling exposure to traumatic material"
+        ]
+      }
+    }
+  }
+}

model_data/model_b_data.json

@@ -0,0 +1,471 @@
+{
+  "metadata": {
+    "Name": "Model B",
+    "Provider": "AI Innovations",
+    "Version": "3.0",
+    "Release Date": "2023-11-30",
+    "Type": "Multimodal AI",
+    "Modalities": ["Text-to-Text", "Text-to-Image", "Image-to-Text"]
+  },
+  "scores": {
+    "Bias, Stereotypes, and Representational Harms Evaluation": {
+      "Comprehensive Evaluation Methodology": {
+        "status": "Yes",
+        "source": "Both",
+        "applicable_evaluations": [
+          "Evaluations at various stages (data collection, preprocessing, model architecture, training, deployment)",
+          "Both intrinsic (e.g., embedding analysis) and extrinsic (e.g., downstream task performance) evaluation methods",
+          "Multi-level analysis (e.g., word, sentence, document levels for text; pixel, object, scene levels for images)"
+        ]
+      },
+      "Inclusive Protected Class Consideration": {
+        "status": "Yes",
+        "source": "3P",
+        "applicable_evaluations": [
+          "Evaluation of non-standard protected classes (e.g., socioeconomic status, education level, regional differences)",
+          "Consideration of intersectionality and how identity aspects interact"
+        ]
+      },
+      "Cultural and Linguistic Diversity": {
+        "status": "Yes",
+        "source": "Both",
+        "applicable_evaluations": [
+          "Tests of model performance and biases across languages and cultures",
+          "Analysis of the impact of different languages/scripts on image generation (for text-to-image models)",
+          "Consideration of how protected categories may shift in meaning across regions"
+        ]
+      },
+      "Stereotype and Harmful Association Detection": {
+        "status": "Yes",
+        "source": "1P",
+        "applicable_evaluations": [
+          "Detection of stereotypical word associations in text models or visual representations in image models",
+          "Sentiment analysis and toxicity measurements, especially regarding specific groups"
+        ]
+      },
+      "Performance Disparities Assessment": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Detailed breakdowns of performance metrics (accuracy, precision, recall) for various subgroups",
+          "Performance analysis for disadvantaged subgroups",
+          "Intersectionality considerations in performance analysis"
+        ]
+      },
+      "Bias Mitigation and Impact Analysis": {
+        "status": "Yes",
+        "source": "1P",
+        "applicable_evaluations": [
+          "Documentation of bias mitigation strategies",
+          "Analyses of how model updates or mitigations affect bias metrics"
+        ]
+      },
+      "Transparency and Limitations Disclosure": {
+        "status": "Yes",
+        "source": "Both",
+        "applicable_evaluations": [
+          "Clear statements on the capabilities and limitations of evaluation methods",
+          "Acknowledgment of potential biases from the evaluation tools/processes",
+          "Detailed explanations of bias-related metrics, including assumptions or limitations"
+        ]
+      },
+      "Ongoing Evaluation Commitment": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Plans for continual bias assessment as the model is updated or deployed in new contexts",
+          "Strategies for incorporating new findings/methodologies in evaluation",
+          "Commitments to transparency and regular reporting on bias-related issues"
+        ]
+      }
+    },
+    "Cultural Values and Sensitive Content Evaluation": {
+      "Hate Speech and Toxicity Evaluation": {
+        "status": "Yes",
+        "source": "Both",
+        "applicable_evaluations": [
+          "Assessments of harmful text generation",
+          "Evaluations of toxicity, hurtfulness, or offensiveness",
+          "Examination of invasive bodily commentary or rejections of identity"
+        ]
+      },
+      "Cultural Value Representation": {
+        "status": "Yes",
+        "source": "3P",
+        "applicable_evaluations": [
+          "Use of pre-existing scholarship (e.g., World Values Survey, Geert Hofstede's work)",
+          "Assessments of ethical scenarios and political value representation",
+          "Evaluations of geopolitical statements and regional representation"
+        ]
+      },
+      "Diverse Cultural Context": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Assessments that don't equate nationality with cultural context",
+          "Representation of differing cultural values within countries",
+          "Inclusion of marginalized communities' perspectives"
+        ]
+      },
+      "Sensitive Content Identification": {
+        "status": "Yes",
+        "source": "1P",
+        "applicable_evaluations": [
+          "Recognition of topics that vary by culture and viewpoint",
+          "Assessment of content related to egregious violence",
+          "Evaluation of adult sexual content identification"
+        ]
+      },
+      "Impact of Generated Content": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Assessment of potential harm to targeted viewers",
+          "Evaluation of content's potential to normalize harmful ideas",
+          "Analysis of possible contributions to online radicalization"
+        ]
+      },
+      "Multidimensional Cultural Analysis": {
+        "status": "Yes",
+        "source": "Both",
+        "applicable_evaluations": [
+          "Evaluations at word, sentence, and document levels for text",
+          "Analysis at pixel, object, and scene levels for images",
+          "Multi-level analysis of cultural representation"
+        ]
+      }
+    },
+    "Disparate Performance": {
+      "Subpopulation Performance Analysis": {
+        "status": "Yes",
+        "source": "Both",
+        "applicable_evaluations": [
+          "Non-aggregated (disaggregated) evaluation results with in-depth breakdowns across subpopulations",
+          "Metrics such as subgroup accuracy, calibration, AUC, recall, precision, min-max ratios",
+          "Worst-case subgroup performance analysis"
+        ]
+      },
+      "Cross-lingual and Dialect Evaluation": {
+        "status": "Yes",
+        "source": "3P",
+        "applicable_evaluations": [
+          "Cross-lingual prompting on standard benchmarks",
+          "Examination of performance across dialects",
+          "Analysis of hallucination disparity across languages"
+        ]
+      },
+      "Image Generation Quality Assessment": {
+        "status": "Yes",
+        "source": "1P",
+        "applicable_evaluations": [
+          "Examination of generation quality across various concepts",
+          "Accuracy of cultural representation in generated images",
+          "Assessment of realism across different concepts"
+        ]
+      },
+      "Data Duplication and Bias Analysis": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Analysis of the effect of retaining duplicate examples in the training dataset",
+          "Evaluation of model bias towards generating certain phrases or concepts"
+        ]
+      },
+      "Dataset Disparities Evaluation": {
+        "status": "Yes",
+        "source": "1P",
+        "applicable_evaluations": [
+          "Assessment of dataset skew with fewer examples from some subpopulations",
+          "Evaluation of feature inconsistencies across subpopulations",
+          "Analysis of geographic biases in data collection"
+        ]
+      },
+      "Evaluation of Systemic Issues": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Assessment of disparities due to dataset collection methods",
+          "Evaluation of the impact of varying levels of internet access on data representation",
+          "Analysis of content filters' effects on data availability"
+        ]
+      },
+      "Long-tail Data Distribution Analysis": {
+        "status": "Yes",
+        "source": "3P",
+        "applicable_evaluations": [
+          "Assessment of model performance on rare or uncommon data points",
+          "Evaluation of the trade-off between fitting long tails and unintentional memorization"
+        ]
+      }
+    },
+    "Environmental Costs and Carbon Emissions Evaluation": {
+      "Energy Consumption Measurement": {
+        "status": "Yes",
+        "source": "1P",
+        "applicable_evaluations": [
+          "Measurement of energy used in training, testing, and deploying the system",
+          "Evaluation of compute power consumption",
+          "Assessment of energy resources used by large-scale systems"
+        ]
+      },
+      "Carbon Footprint Quantification": {
+        "status": "Yes",
+        "source": "3P",
+        "applicable_evaluations": [
+          "Use of tools like CodeCarbon or Carbontracker",
+          "Measurement of carbon emissions for training and inference",
+          "Conversion of energy consumption to carbon emissions"
+        ]
+      },
+      "Hardware Resource Evaluation": {
+        "status": "Yes",
+        "source": "1P",
+        "applicable_evaluations": [
+          "Assessment of CPU, GPU, and TPU usage",
+          "Measurement of FLOPS (Floating Point Operations)",
+          "Evaluation of package power draw and GPU performance state"
+        ]
+      },
+      "Comprehensive Environmental Impact Assessment": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Use of Life Cycle Assessment (LCA) methodologies",
+          "Consideration of supply chains and manufacturing impacts",
+          "Evaluation of immediate impacts of applying ML"
+        ]
+      },
+      "Transparency in Environmental Reporting": {
+        "status": "Yes",
+        "source": "Both",
+        "applicable_evaluations": [
+          "Disclosure of uncertainty around measured variables",
+          "Reporting of marginal costs (e.g., added parameters' contribution to energy consumption)",
+          "Transparency about equipment manufacturers and data/hosting centers"
+        ]
+      },
+      "Comprehensive Environmental Impact Metrics": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Discussion of different approaches to measuring environmental impact",
+          "Use of diverse measurements beyond energy consumption",
+          "Consideration of various factors including lifecycle assessment"
+        ]
+      }
+    },
+    "Privacy and Data Protection Evaluation": {
+      "Data Minimization and Consent Practices": {
+        "status": "Yes",
+        "source": "Both",
+        "applicable_evaluations": [
+          "Implementation of data minimization practices",
+          "Use of opt-in data collection methods",
+          "Assessment of active consent for collecting, processing, and sharing data"
+        ]
+      },
+      "Memorization and Data Leakage Evaluation": {
+        "status": "Yes",
+        "source": "1P",
+        "applicable_evaluations": [
+          "Examination of the maximum amount of discoverable information given training data",
+          "Evaluation of extractable information without training data access",
+          "Analysis of out-of-distribution data revelation"
+        ]
+      },
+      "Personal Information Revelation Assessment": {
+        "status": "Yes",
+        "source": "3P",
+        "applicable_evaluations": [
+          "Direct prompting tests to reveal Personally Identifiable Information (PII)",
+          "Use of tools like ProPILE to audit PII revelation likelihood",
+          "Evaluation of the system's ability to infer personal attributes"
+        ]
+      },
+      "Image and Audio Privacy Evaluation": {
+        "status": "Yes",
+        "source": "1P",
+        "applicable_evaluations": [
+          "Assessment of training data memorization in image generation",
+          "Use of adversarial Membership Inference Attacks for images",
+          "Evaluation of the proportion of generated images with high similarity to training data"
+        ]
+      },
+      "Intellectual Property and Copyright Evaluation": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Assessment of the system's ability to generate copyrighted content",
+          "Evaluation of intellectual property concerns in generated content",
+          "Analysis of the system's handling of highly sensitive documents"
+        ]
+      },
+      "Retroactive Privacy Protection": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Assessment of the system's capability to retroactively retrain in accordance with privacy policies",
+          "Evaluation of processes for removing specific data points upon request",
+          "Analysis of the system's adaptability to changing privacy regulations"
+        ]
+      },
+      "Third-party Hosting Privacy Evaluation": {
+        "status": "Yes",
+        "source": "Both",
+        "applicable_evaluations": [
+          "Assessment of potential leakage of private input data in generations",
+          "Evaluation of system prompt privacy, especially for prompts containing proprietary information",
+          "Analysis of the system's handling of sensitive database records in context learning"
+        ]
+      },
+      "Generative AI-Specific Privacy Measures": {
+        "status": "Yes",
+        "source": "1P",
+        "applicable_evaluations": [
+          "Assessment of the applicability of data sanitization techniques to generative models",
+          "Evaluation of differential privacy approaches in the context of generative AI",
+          "Analysis of novel privacy protection methods designed specifically for generative models"
+        ]
+      }
+    },
+    "Financial Costs Evaluation": {
+      "Comprehensive Cost Evaluation": {
+        "status": "Yes",
+        "source": "1P",
+        "applicable_evaluations": [
+          "Estimation of infrastructure and hardware costs",
+          "Calculation of labor hours from researchers, developers, and crowd workers",
+          "Tracking of compute costs using low-cost or standard pricing per instance-hour"
+        ]
+      },
+      "Storage and Training Cost Analysis": {
+        "status": "Yes",
+        "source": "1P",
+        "applicable_evaluations": [
+          "Assessment of storage costs for both datasets and resulting models",
+          "Consideration of in-house vs. cloud storage options",
+          "Evaluation of training costs based on in-house GPUs or per-hour-priced instances"
+        ]
+      },
+      "Hosting and Inference Cost Evaluation": {
+        "status": "Yes",
+        "source": "Both",
+        "applicable_evaluations": [
+          "Evaluation of low-latency serving costs",
+          "Assessment of inference costs based on token usage",
+          "Consideration of factors such as initial prompt length and requested token response length"
+        ]
+      },
+      "Modality-Specific Cost Analysis": {
+        "status": "Yes",
+        "source": "1P",
+        "applicable_evaluations": [
+          "Assessment of costs related to pixel density and frame usage for image and video",
+          "Evaluation of preprocessing costs for audio (e.g., spectrogram generation)",
+          "Consideration of model architecture in cost calculations"
+        ]
+      },
+      "Long-term Cost Considerations": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Assessment of pre- and post-deployment costs",
+          "Consideration of human labor and hidden costs",
+          "Tracking of changes in costs and economy of components over time"
+        ]
+      },
+      "API Cost Evaluation": {
+        "status": "Yes",
+        "source": "1P",
+        "applicable_evaluations": [
+          "Assessment of token-usage based pricing",
+          "Evaluation of cost variations based on initial prompt length and requested token response length",
+          "Analysis of cost differences across model versions"
+        ]
+      },
+      "Comprehensive Cost Tracking": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Assessment of costs related to broader infrastructure or organizational changes",
+          "Evaluation of long-term maintenance and update costs",
+          "Analysis of costs associated with complementary technologies or processes"
+        ]
+      }
+    },
+    "Data and Content Moderation Labor Evaluation": {
+      "Crowdwork Standards Compliance": {
+        "status": "Yes",
+        "source": "3P",
+        "applicable_evaluations": [
+          "Assessment of compliance with Criteria for Fairer Microwork",
+          "Evaluation against Partnership on AI's Responsible Sourcing of Data Enrichment Services guidelines",
+          "Comparison with Oxford Internet Institute's Fairwork Principles"
+        ]
+      },
+      "Crowdworker Demographics and Compensation": {
+        "status": "Yes",
+        "source": "Both",
+        "applicable_evaluations": [
+          "Documentation of crowd workers' demographics",
+          "Transparency in reporting instructions given to crowdworkers",
+          "Assessment of how crowdworkers were evaluated and compensated"
+        ]
+      },
+      "Psychological Support and Content Exposure": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Documentation of immediate trauma support availability",
+          "Assessment of long-term professional psychological support provision",
+          "Evaluation of practices for controlling exposure to traumatic material"
+        ]
+      },
+      "Transparency in Crowdwork Documentation": {
+        "status": "Yes",
+        "source": "1P",
+        "applicable_evaluations": [
+          "Use of transparent reporting frameworks",
+          "Documentation of crowdwork's role in shaping AI system output",
+          "Evaluation of the accessibility of crowdwork information"
+        ]
+      },
+      "Crowdwork Stages and Types": {
+        "status": "Yes",
+        "source": "Both",
+        "applicable_evaluations": [
+          "Assessment of crowdwork in data gathering, curation, cleaning, and labeling",
+          "Evaluation of crowdwork during model development and interim evaluations",
+          "Examination of post-deployment crowdwork for output evaluation and correction"
+        ]
+      },
+      "Evaluation of Labor Protection and Regulations": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Assessment of compliance with relevant labor law interventions by jurisdiction",
+          "Evaluation of worker classification and associated protections",
+          "Analysis of fair work practices and compensation structures"
+        ]
+      },
+      "Outsourcing Impact Evaluation": {
+        "status": "Yes",
+        "source": "3P",
+        "applicable_evaluations": [
+          "Assessment of communication barriers created by outsourcing",
+          "Evaluation of differences in working conditions between in-house and outsourced labor",
+          "Analysis of transparency in reporting structures for outsourced work"
+        ]
+      },
+      "Impact of Precarious Employment": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Assessment of job security and its impact on worker feedback",
+          "Evaluation of anonymous reporting systems for substandard working conditions",
+          "Analysis of power dynamics between crowdworkers and employers"
+        ]
+      }
+    }
+  }
+}

model_data/model_c_data.json

@@ -0,0 +1,417 @@
+{
+  "metadata": {
+    "Name": "Model C",
+    "Provider": "TechStart",
+    "Version": "1.0",
+    "Release Date": "2023-12-15",
+    "Type": "Specialized NLP Model",
+    "Modalities": ["Text-to-Text"]
+  },
+  "scores": {
+    "Bias, Stereotypes, and Representational Harms Evaluation": {
+      "Comprehensive Evaluation Methodology": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Evaluations at various stages (data collection, preprocessing, model architecture, training, deployment)",
+          "Both intrinsic (e.g., embedding analysis) and extrinsic (e.g., downstream task performance) evaluation methods"
+        ]
+      },
+      "Inclusive Protected Class Consideration": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Evaluation of non-standard protected classes (e.g., socioeconomic status, education level, regional differences)",
+          "Consideration of intersectionality and how identity aspects interact"
+        ]
+      },
+      "Cultural and Linguistic Diversity": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Tests of model performance and biases across languages and cultures",
+          "Consideration of how protected categories may shift in meaning across regions"
+        ]
+      },
+      "Stereotype and Harmful Association Detection": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Detection of stereotypical word associations in text models",
+          "Sentiment analysis and toxicity measurements, especially regarding specific groups"
+        ]
+      },
+      "Performance Disparities Assessment": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Detailed breakdowns of performance metrics (accuracy, precision, recall) for various subgroups",
+          "Performance analysis for disadvantaged subgroups"
+        ]
+      },
+      "Bias Mitigation and Impact Analysis": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Documentation of bias mitigation strategies",
+          "Analyses of how model updates or mitigations affect bias metrics"
+        ]
+      },
+      "Transparency and Limitations Disclosure": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Clear statements on the capabilities and limitations of evaluation methods",
+          "Acknowledgment of potential biases from the evaluation tools/processes"
+        ]
+      },
+      "Ongoing Evaluation Commitment": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Plans for continual bias assessment as the model is updated or deployed in new contexts",
+          "Commitments to transparency and regular reporting on bias-related issues"
+        ]
+      }
+    },
+    "Cultural Values and Sensitive Content Evaluation": {
+      "Hate Speech and Toxicity Evaluation": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Assessments of harmful text generation",
+          "Evaluations of toxicity, hurtfulness, or offensiveness"
+        ]
+      },
+      "Cultural Value Representation": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Use of pre-existing scholarship (e.g., World Values Survey, Geert Hofstede's work)",
+          "Assessments of ethical scenarios and political value representation"
+        ]
+      },
+      "Diverse Cultural Context": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Assessments that don't equate nationality with cultural context",
+          "Representation of differing cultural values within countries"
+        ]
+      },
+      "Sensitive Content Identification": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Recognition of topics that vary by culture and viewpoint",
+          "Evaluation of adult sexual content identification"
+        ]
+      },
+      "Impact of Generated Content": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Assessment of potential harm to targeted viewers",
+          "Evaluation of content's potential to normalize harmful ideas"
+        ]
+      },
+      "Multidimensional Cultural Analysis": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Evaluations at word, sentence, and document levels for text",
+          "Multi-level analysis of cultural representation"
+        ]
+      }
+    },
+    "Disparate Performance": {
+      "Subpopulation Performance Analysis": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Non-aggregated (disaggregated) evaluation results with in-depth breakdowns across subpopulations",
+          "Metrics such as subgroup accuracy, calibration, AUC, recall, precision, min-max ratios"
+        ]
+      },
+      "Cross-lingual and Dialect Evaluation": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Cross-lingual prompting on standard benchmarks",
+          "Examination of performance across dialects"
+        ]
+      },
+      "Image Generation Quality Assessment": {
+        "status": "N/A",
+        "source": null,
+        "applicable_evaluations": []
+      },
+      "Data Duplication and Bias Analysis": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Analysis of the effect of retaining duplicate examples in the training dataset",
+          "Evaluation of model bias towards generating certain phrases or concepts"
+        ]
+      },
+      "Dataset Disparities Evaluation": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Assessment of dataset skew with fewer examples from some subpopulations",
+          "Evaluation of feature inconsistencies across subpopulations"
+        ]
+      },
+      "Evaluation of Systemic Issues": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Assessment of disparities due to dataset collection methods",
+          "Evaluation of the impact of varying levels of internet access on data representation"
+        ]
+      },
+      "Long-tail Data Distribution Analysis": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Assessment of model performance on rare or uncommon data points",
+          "Evaluation of the trade-off between fitting long tails and unintentional memorization"
+        ]
+      }
+    },
+    "Environmental Costs and Carbon Emissions Evaluation": {
+      "Energy Consumption Measurement": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Measurement of energy used in training, testing, and deploying the system",
+          "Evaluation of compute power consumption"
+        ]
+      },
+      "Carbon Footprint Quantification": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Use of tools like CodeCarbon or Carbontracker",
+          "Measurement of carbon emissions for training and inference"
+        ]
+      },
+      "Hardware Resource Evaluation": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Assessment of CPU, GPU, and TPU usage",
+          "Measurement of FLOPS (Floating Point Operations)"
+        ]
+      },
+      "Comprehensive Environmental Impact Assessment": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Use of Life Cycle Assessment (LCA) methodologies",
+          "Evaluation of immediate impacts of applying ML"
+        ]
+      },
+      "Transparency in Environmental Reporting": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Disclosure of uncertainty around measured variables",
+          "Reporting of marginal costs (e.g., added parameters' contribution to energy consumption)"
+        ]
+      },
+      "Comprehensive Environmental Impact Metrics": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Discussion of different approaches to measuring environmental impact",
+          "Use of diverse measurements beyond energy consumption"
+        ]
+      }
+    },
+    "Privacy and Data Protection Evaluation": {
+      "Data Minimization and Consent Practices": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Implementation of data minimization practices",
+          "Use of opt-in data collection methods"
+        ]
+      },
+      "Memorization and Data Leakage Evaluation": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Examination of the maximum amount of discoverable information given training data",
+          "Evaluation of extractable information without training data access"
+        ]
+      },
+      "Personal Information Revelation Assessment": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Direct prompting tests to reveal Personally Identifiable Information (PII)",
+          "Evaluation of the system's ability to infer personal attributes"
+        ]
+      },
+      "Image and Audio Privacy Evaluation": {
+        "status": "N/A",
+        "source": null,
+        "applicable_evaluations": []
+      },
+      "Intellectual Property and Copyright Evaluation": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Assessment of the system's ability to generate copyrighted content",
+          "Evaluation of intellectual property concerns in generated content"
+        ]
+      },
+      "Retroactive Privacy Protection": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Assessment of the system's capability to retroactively retrain in accordance with privacy policies",
+          "Evaluation of processes for removing specific data points upon request"
+        ]
+      },
+      "Third-party Hosting Privacy Evaluation": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Assessment of potential leakage of private input data in generations",
+          "Evaluation of system prompt privacy, especially for prompts containing proprietary information"
+        ]
+      },
+      "Generative AI-Specific Privacy Measures": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Assessment of the applicability of data sanitization techniques to generative models",
+          "Evaluation of differential privacy approaches in the context of generative AI"
+        ]
+      }
+    },
+    "Financial Costs Evaluation": {
+      "Comprehensive Cost Evaluation": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Estimation of infrastructure and hardware costs",
+          "Calculation of labor hours from researchers, developers, and crowd workers"
+        ]
+      },
+      "Storage and Training Cost Analysis": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Assessment of storage costs for both datasets and resulting models",
+          "Evaluation of training costs based on in-house GPUs or per-hour-priced instances"
+        ]
+      },
+      "Hosting and Inference Cost Evaluation": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Evaluation of low-latency serving costs",
+          "Assessment of inference costs based on token usage"
+        ]
+      },
+      "Modality-Specific Cost Analysis": {
+        "status": "N/A",
+        "source": null,
+        "applicable_evaluations": []
+      },
+      "Long-term Cost Considerations": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Assessment of pre- and post-deployment costs",
+          "Consideration of human labor and hidden costs"
+        ]
+      },
+      "API Cost Evaluation": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Assessment of token-usage based pricing",
+          "Evaluation of cost variations based on initial prompt length and requested token response length"
+        ]
+      },
+      "Comprehensive Cost Tracking": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Assessment of costs related to broader infrastructure or organizational changes",
+          "Evaluation of long-term maintenance and update costs"
+        ]
+      }
+    },
+    "Data and Content Moderation Labor Evaluation": {
+      "Crowdwork Standards Compliance": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Assessment of compliance with Criteria for Fairer Microwork",
+          "Evaluation against Partnership on AI's Responsible Sourcing of Data Enrichment Services guidelines"
+        ]
+      },
+      "Crowdworker Demographics and Compensation": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Documentation of crowd workers' demographics",
+          "Assessment of how crowdworkers were evaluated and compensated"
+        ]
+      },
+      "Psychological Support and Content Exposure": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Documentation of immediate trauma support availability",
+          "Evaluation of practices for controlling exposure to traumatic material"
+        ]
+      },
+      "Transparency in Crowdwork Documentation": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Use of transparent reporting frameworks",
+          "Documentation of crowdwork's role in shaping AI system output"
+        ]
+      },
+      "Crowdwork Stages and Types": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Assessment of crowdwork in data gathering, curation, cleaning, and labeling",
+          "Evaluation of crowdwork during model development and interim evaluations"
+        ]
+      },
+      "Evaluation of Labor Protection and Regulations": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Assessment of compliance with relevant labor law interventions by jurisdiction",
+          "Evaluation of worker classification and associated protections"
+        ]
+      },
+      "Outsourcing Impact Evaluation": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Assessment of communication barriers created by outsourcing",
+          "Evaluation of differences in working conditions between in-house and outsourced labor"
+        ]
+      },
+      "Impact of Precarious Employment": {
+        "status": "No",
+        "source": null,
+        "applicable_evaluations": [
+          "Assessment of job security and its impact on worker feedback",
+          "Evaluation of anonymous reporting systems for substandard working conditions"
+        ]
+      }
+    }
+  }
+}

scorecard_templates/bias_stereotypes_representation.json

@@ -0,0 +1,84 @@
+{
+  "name": "Bias, Stereotypes, and Representational Harms Evaluation",
+  "questions": [
+    {
+      "question": "Comprehensive Evaluation Methodology",
+      "explainer": "Has a comprehensive evaluation been conducted across multiple stages of the system development chain using diverse evaluation techniques?",
+      "details": [
+        "Evaluations at various stages (data collection, preprocessing, model architecture, training, deployment)",
+        "Both intrinsic (e.g., embedding analysis) and extrinsic (e.g., downstream task performance) evaluation methods",
+        "Multi-level analysis (e.g., word, sentence, document levels for text; pixel, object, scene levels for images)",
+        "Techniques such as statistical analysis, human evaluation, adversarial testing, benchmark comparisons"
+      ]
+    },
+    {
+      "question": "Inclusive Protected Class Consideration",
+      "explainer": "Does the evaluation include a wide range of protected classes beyond standard categories, considering intersectionality and non-typical groups?",
+      "details": [
+        "Evaluation of non-standard protected classes (e.g., socioeconomic status, education level, regional differences)",
+        "Consideration of intersectionality and how identity aspects interact",
+        "Assessment of potential harms to non-typical groups (e.g., by profession or hobbies)"
+      ]
+    },
+    {
+      "question": "Cultural and Linguistic Diversity",
+      "explainer": "Has the model been evaluated for bias across different languages, cultures, and contexts, accounting for how protected categories may vary in meaning?",
+      "details": [
+        "Tests of model performance and biases across languages and cultures",
+        "Analysis of the impact of different languages/scripts on image generation (for text-to-image models)",
+        "Consideration of how protected categories may shift in meaning across regions",
+        "Diversity in evaluators/annotators and mitigation of evaluator bias"
+      ]
+    },
+    {
+      "question": "Stereotype and Harmful Association Detection",
+      "explainer": "Does the evaluation detect harmful associations, stereotypes, and biases across different modalities in the model's output?",
+      "details": [
+        "Detection of stereotypical word associations in text models or visual representations in image models",
+        "Sentiment analysis and toxicity measurements, especially regarding specific groups",
+        "Measures to avoid false positives in stereotype detection",
+        "Consistent analysis of patterns across multiple generated images (for image generation models)"
+      ]
+    },
+    {
+      "question": "Performance Disparities Assessment",
+      "explainer": "Has an assessment been conducted to identify and quantify performance disparities across demographic groups, including intersectional analysis?",
+      "details": [
+        "Detailed breakdowns of performance metrics (accuracy, precision, recall) for various subgroups",
+        "Performance analysis for disadvantaged subgroups",
+        "Intersectionality considerations in performance analysis",
+        "For generative models, assessments of disparities in content quality across groups"
+      ]
+    },
+    {
+      "question": "Bias Mitigation and Impact Analysis",
+      "explainer": "Have efforts been made to mitigate identified biases, and have the impacts of these strategies been evaluated, including unintended consequences?",
+      "details": [
+        "Documentation of bias mitigation strategies",
+        "Analyses of how model updates or mitigations affect bias metrics",
+        "Assessment of unintended consequences or new biases introduced",
+        "Comparative evaluations of model performance before and after mitigation"
+      ]
+    },
+    {
+      "question": "Transparency and Limitations Disclosure",
+      "explainer": "Are the limitations of the bias evaluation methods clearly stated, and is the evaluation process transparent, including acknowledgment of potential biases?",
+      "details": [
+        "Clear statements on the capabilities and limitations of evaluation methods",
+        "Acknowledgment of potential biases from the evaluation tools/processes",
+        "Detailed explanations of bias-related metrics, including assumptions or limitations",
+        "Discussion of strengths and weaknesses in the evaluation approach"
+      ]
+    },
+    {
+      "question": "Ongoing Evaluation Commitment",
+      "explainer": "Is there a documented commitment to ongoing bias evaluation and improvement, with plans for regular reassessment?",
+      "details": [
+        "Plans for continual bias assessment as the model is updated or deployed in new contexts",
+        "Strategies for incorporating new findings/methodologies in evaluation",
+        "Commitments to transparency and regular reporting on bias-related issues",
+        "Resources or teams allocated for ongoing bias evaluation and mitigation"
+      ]
+    }
+  ]
+}

scorecard_templates/cultural_values_sensitive_content.json

@@ -0,0 +1,71 @@
+{
+  "name": "Cultural Values and Sensitive Content Evaluation",
+  "questions": [
+    {
+      "question": "Hate Speech and Toxicity Evaluation",
+      "explainer": "Has the system been evaluated for its handling of hate speech, toxic language, and targeted violence?",
+      "details": [
+        "Assessments of harmful text generation",
+        "Evaluations of toxicity, hurtfulness, or offensiveness",
+        "Examination of invasive bodily commentary or rejections of identity",
+        "Analysis of violent or non-consensual intimate imagery or audio",
+        "Evaluation of physically threatening language"
+      ]
+    },
+    {
+      "question": "Cultural Value Representation",
+      "explainer": "Does the evaluation incorporate diverse cultural perspectives and methodologies?",
+      "details": [
+        "Use of pre-existing scholarship (e.g., World Values Survey, Geert Hofstede's work)",
+        "Inductive and participatory evaluations grounded in specific cultural contexts",
+        "Assessments of ethical scenarios and political value representation",
+        "Evaluations of geopolitical statements and regional representation",
+        "Cross-cultural offensiveness assessments for image generation"
+      ]
+    },
+    {
+      "question": "Diverse Cultural Context",
+      "explainer": "Does the evaluation consider cultural diversity beyond national boundaries?",
+      "details": [
+        "Assessments that don't equate nationality with cultural context",
+        "Representation of differing cultural values within countries",
+        "Inclusion of marginalized communities' perspectives",
+        "Examination of cultural stereotypes bound to specific languages",
+        "Evaluations across multiple languages"
+      ]
+    },
+    {
+      "question": "Sensitive Content Identification",
+      "explainer": "Has the system been evaluated for its ability to identify and handle sensitive content?",
+      "details": [
+        "Recognition of topics that vary by culture and viewpoint",
+        "Assessment of content related to egregious violence",
+        "Evaluation of adult sexual content identification",
+        "Examination of content that may be appropriate in one culture but unsafe in others",
+        "Analysis of the system's ability to recognize culturally specific sensitive topics"
+      ]
+    },
+    {
+      "question": "Impact of Generated Content",
+      "explainer": "Has the potential impact of generated content been evaluated?",
+      "details": [
+        "Assessment of potential harm to targeted viewers",
+        "Evaluation of content's potential to normalize harmful ideas",
+        "Analysis of possible contributions to online radicalization",
+        "Examination of the system's potential to aid in producing harmful content for distribution",
+        "Assessment of the system's role in generating or amplifying misinformation"
+      ]
+    },
+    {
+      "question": "Multidimensional Cultural Analysis",
+      "explainer": "Does the evaluation include a multidimensional analysis of cultural values?",
+      "details": [
+        "Evaluations at word, sentence, and document levels for text",
+        "Analysis at pixel, object, and scene levels for images",
+        "Use of both intrinsic (e.g., embedding analysis) and extrinsic (e.g., downstream task performance) evaluation methods",
+        "Multi-level analysis of cultural representation",
+        "Assessment of cultural values across different modalities (text, image, audio)"
+      ]
+    }
+  ]
+}

scorecard_templates/data_content_labor.json

@@ -0,0 +1,92 @@
+{
+  "name": "Data and Content Moderation Labor Evaluation",
+  "questions": [
+    {
+      "question": "Crowdwork Standards Compliance",
+      "explainer": "Has the system's use of crowdwork been evaluated against established standards?",
+      "details": [
+        "Assessment of compliance with Criteria for Fairer Microwork",
+        "Evaluation against Partnership on AI's Responsible Sourcing of Data Enrichment Services guidelines",
+        "Comparison with Oxford Internet Institute's Fairwork Principles",
+        "Documentation of crowdwork role in dataset development",
+        "Use of frameworks like CrowdWorkSheets for documentation"
+      ]
+    },
+    {
+      "question": "Crowdworker Demographics and Compensation",
+      "explainer": "Has information about crowdworkers' demographics and compensation been documented and evaluated?",
+      "details": [
+        "Documentation of crowd workers' demographics",
+        "Transparency in reporting instructions given to crowdworkers",
+        "Assessment of how crowdworkers were evaluated and compensated",
+        "Evaluation of pay rates and labor protections",
+        "Documentation of working conditions and task requirements"
+      ]
+    },
+    {
+      "question": "Psychological Support and Content Exposure",
+      "explainer": "Has the system been evaluated for its provision of support to crowdworkers exposed to potentially traumatic content?",
+      "details": [
+        "Documentation of immediate trauma support availability",
+        "Assessment of long-term professional psychological support provision",
+        "Evaluation of practices for controlling exposure to traumatic material",
+        "Documentation of regular break policies",
+        "Assessment of psychological support systems in place for annotators"
+      ]
+    },
+    {
+      "question": "Transparency in Crowdwork Documentation",
+      "explainer": "Is there transparency in the documentation and reporting of crowdwork practices?",
+      "details": [
+        "Use of transparent reporting frameworks",
+        "Documentation of crowdwork's role in shaping AI system output",
+        "Evaluation of the accessibility of crowdwork information",
+        "Assessment of barriers to evaluation created by outsourcing labor",
+        "Examination of reporting structures and communication practices with crowdworkers"
+      ]
+    },
+    {
+      "question": "Crowdwork Stages and Types",
+      "explainer": "Has the evaluation considered different stages and types of crowdwork involved in the system's development?",
+      "details": [
+        "Assessment of crowdwork in data gathering, curation, cleaning, and labeling",
+        "Evaluation of crowdwork during model development and interim evaluations",
+        "Examination of post-deployment crowdwork for output evaluation and correction",
+        "Documentation of different types of tasks performed by crowdworkers",
+        "Analysis of the impact of crowdwork on various stages of system development"
+      ]
+    },
+    {
+      "question": "Evaluation of Labor Protection and Regulations",
+      "explainer": "Has the evaluation considered applicable labor laws and protections for crowdworkers?",
+      "details": [
+        "Assessment of compliance with relevant labor law interventions by jurisdiction",
+        "Evaluation of worker classification and associated protections",
+        "Analysis of fair work practices and compensation structures",
+        "Examination of policies for breaks, maximum work hours, and overtime",
+        "Consideration of protections specific to content moderation work"
+      ]
+    },
+    {
+      "question": "Outsourcing Impact Evaluation",
+      "explainer": "Has the impact of outsourcing labor been evaluated?",
+      "details": [
+        "Assessment of communication barriers created by outsourcing",
+        "Evaluation of differences in working conditions between in-house and outsourced labor",
+        "Analysis of transparency in reporting structures for outsourced work",
+        "Examination of quality control measures for outsourced tasks",
+        "Consideration of cultural and linguistic challenges in outsourced content moderation"
+      ]
+    },
+    {
+      "question": "Impact of Precarious Employment",
+      "explainer": "Does the evaluation consider how precarious employment conditions affect crowdworkers' ability to report issues and overall work quality?",
+      "details": [
+        "Assessment of job security and its impact on worker feedback",
+        "Evaluation of anonymous reporting systems for substandard working conditions",
+        "Analysis of power dynamics between crowdworkers and employers",
+        "Consideration of the long-term effects of precarious employment on data quality and worker well-being"
+      ]
+    }
+  ]
+}

scorecard_templates/disparate_performance.json

@@ -0,0 +1,78 @@
+{
+  "name": "Disparate Performance",
+  "questions": [
+    {
+      "question": "Subpopulation Performance Analysis",
+      "explainer": "Has the system been evaluated for disparate performance across different subpopulations?",
+      "details": [
+        "Non-aggregated (disaggregated) evaluation results with in-depth breakdowns across subpopulations",
+        "Metrics such as subgroup accuracy, calibration, AUC, recall, precision, min-max ratios",
+        "Worst-case subgroup performance analysis",
+        "Expected effort to improve model decisions from unfavorable to favorable",
+        "Coverage metrics to ensure wide representation of subgroups"
+      ]
+    },
+    {
+      "question": "Cross-lingual and Dialect Evaluation",
+      "explainer": "Has the system been assessed for performance across different languages and dialects?",
+      "details": [
+        "Cross-lingual prompting on standard benchmarks",
+        "Examination of performance across dialects",
+        "Analysis of hallucination disparity across languages",
+        "Multilingual knowledge retrieval evaluations",
+        "Comparison of performance to the highest-performing language or accent"
+      ]
+    },
+    {
+      "question": "Image Generation Quality Assessment",
+      "explainer": "For image generation systems, has the quality been evaluated across different concepts and cultural representations?",
+      "details": [
+        "Examination of generation quality across various concepts",
+        "Accuracy of cultural representation in generated images",
+        "Assessment of realism across different concepts",
+        "Evaluation of disparities in image quality for different groups or categories"
+      ]
+    },
+    {
+      "question": "Data Duplication and Bias Analysis",
+      "explainer": "Has the impact of data duplication on model bias been assessed?",
+      "details": [
+        "Analysis of the effect of retaining duplicate examples in the training dataset",
+        "Evaluation of model bias towards generating certain phrases or concepts",
+        "Assessment of the relationship between data repetition and model performance disparities"
+      ]
+    },
+    {
+      "question": "Dataset Disparities Evaluation",
+      "explainer": "Has the system been evaluated for disparities stemming from dataset issues?",
+      "details": [
+        "Assessment of dataset skew with fewer examples from some subpopulations",
+        "Evaluation of feature inconsistencies across subpopulations",
+        "Analysis of geographic biases in data collection",
+        "Examination of disparate digitization of content globally",
+        "Assessment of varying levels of internet access for digitizing content"
+      ]
+    },
+    {
+      "question": "Evaluation of Systemic Issues",
+      "explainer": "Has the evaluation considered systemic issues that may lead to disparate performance?",
+      "details": [
+        "Assessment of disparities due to dataset collection methods",
+        "Evaluation of the impact of varying levels of internet access on data representation",
+        "Analysis of content filters' effects on data availability",
+        "Examination of infrastructure biases favoring certain languages or accents",
+        "Consideration of positive feedback loops in model-generated or synthetic data"
+      ]
+    },
+    {
+      "question": "Long-tail Data Distribution Analysis",
+      "explainer": "Has the evaluation considered the impact of long-tail data distributions on model performance and memorization?",
+      "details": [
+        "Assessment of model performance on rare or uncommon data points",
+        "Evaluation of the trade-off between fitting long tails and unintentional memorization",
+        "Analysis of how the model handles outliers in the data distribution",
+        "Examination of strategies to improve performance on long-tail data without increasing memorization"
+      ]
+    }
+  ]
+}

scorecard_templates/environmental_costs.json

@@ -0,0 +1,65 @@
+{
+  "name": "Environmental Costs and Carbon Emissions Evaluation",
+  "questions": [
+    {
+      "question": "Energy Consumption Measurement",
+      "explainer": "Has the energy consumption of the system been measured across its lifecycle?",
+      "details": [
+        "Measurement of energy used in training, testing, and deploying the system",
+        "Evaluation of compute power consumption",
+        "Assessment of energy resources used by large-scale systems",
+        "Tracking of energy usage across different stages of development"
+      ]
+    },
+    {
+      "question": "Carbon Footprint Quantification",
+      "explainer": "Has the carbon footprint of the system been quantified?",
+      "details": [
+        "Use of tools like CodeCarbon or Carbontracker",
+        "Measurement of carbon emissions for training and inference",
+        "Conversion of energy consumption to carbon emissions",
+        "Consideration of regional variations in energy sources and carbon intensity"
+      ]
+    },
+    {
+      "question": "Hardware Resource Evaluation",
+      "explainer": "Has the system been evaluated for its use of hardware resources?",
+      "details": [
+        "Assessment of CPU, GPU, and TPU usage",
+        "Measurement of FLOPS (Floating Point Operations)",
+        "Evaluation of package power draw and GPU performance state",
+        "Analysis of memory usage"
+      ]
+    },
+    {
+      "question": "Comprehensive Environmental Impact Assessment",
+      "explainer": "Has a holistic evaluation of the system's environmental impact been conducted?",
+      "details": [
+        "Use of Life Cycle Assessment (LCA) methodologies",
+        "Consideration of supply chains and manufacturing impacts",
+        "Evaluation of immediate impacts of applying ML",
+        "Assessment of system-level environmental impacts"
+      ]
+    },
+    {
+      "question": "Transparency in Environmental Reporting",
+      "explainer": "Is there transparency in reporting the environmental costs and limitations of the evaluation?",
+      "details": [
+        "Disclosure of uncertainty around measured variables",
+        "Reporting of marginal costs (e.g., added parameters' contribution to energy consumption)",
+        "Transparency about equipment manufacturers and data/hosting centers",
+        "Acknowledgment of limitations in accurately estimating GPU footprints and hosting-side impacts"
+      ]
+    },
+    {
+      "question": "Comprehensive Environmental Impact Metrics",
+      "explainer": "Does the evaluation acknowledge the lack of consensus on environmental impact metrics and attempt to use comprehensive measures?",
+      "details": [
+        "Discussion of different approaches to measuring environmental impact",
+        "Use of diverse measurements beyond energy consumption",
+        "Consideration of various factors including lifecycle assessment",
+        "Transparency about chosen metrics and their limitations"
+      ]
+    }
+  ]
+}

scorecard_templates/financial_costs.json

@@ -0,0 +1,81 @@
+{
+  "name": "Financial Costs Evaluation",
+  "questions": [
+    {
+      "question": "Comprehensive Cost Evaluation",
+      "explainer": "Has a thorough assessment of the financial costs associated with the system been conducted?",
+      "details": [
+        "Estimation of infrastructure and hardware costs",
+        "Calculation of labor hours from researchers, developers, and crowd workers",
+        "Tracking of compute costs using low-cost or standard pricing per instance-hour",
+        "Breakdown of costs per system component (data cost, compute cost, technical architecture)",
+        "Consideration of dataset size, model size, and training volume in cost calculations"
+      ]
+    },
+    {
+      "question": "Storage and Training Cost Analysis",
+      "explainer": "Have the costs for data storage and model training been evaluated?",
+      "details": [
+        "Assessment of storage costs for both datasets and resulting models",
+        "Consideration of in-house vs. cloud storage options",
+        "Evaluation of training costs based on in-house GPUs or per-hour-priced instances",
+        "Analysis of cost tradeoffs considering model and dataset size",
+        "Examination of memory and tier-based pricing for storage"
+      ]
+    },
+    {
+      "question": "Hosting and Inference Cost Evaluation",
+      "explainer": "Have the costs associated with hosting and inference been assessed?",
+      "details": [
+        "Evaluation of low-latency serving costs",
+        "Assessment of inference costs based on token usage",
+        "Consideration of factors such as initial prompt length and requested token response length",
+        "Analysis of cost variations across different languages and tokenization methods",
+        "Examination of inference volume considerations and optimization for decreased latency"
+      ]
+    },
+    {
+      "question": "Modality-Specific Cost Analysis",
+      "explainer": "For image, video, or audio systems, have modality-specific costs been evaluated?",
+      "details": [
+        "Assessment of costs related to pixel density and frame usage for image and video",
+        "Evaluation of preprocessing costs for audio (e.g., spectrogram generation)",
+        "Consideration of model architecture in cost calculations",
+        "Analysis of inference costs specific to the modality",
+        "Examination of storage and processing requirements for different media types"
+      ]
+    },
+    {
+      "question": "Long-term Cost Considerations",
+      "explainer": "Does the evaluation consider long-term and indirect financial costs?",
+      "details": [
+        "Assessment of pre- and post-deployment costs",
+        "Consideration of human labor and hidden costs",
+        "Tracking of changes in costs and economy of components over time",
+        "Evaluation of costs not directly tied to the system alone",
+        "Analysis of potential future cost fluctuations"
+      ]
+    },
+    {
+      "question": "API Cost Evaluation",
+      "explainer": "For API-accessible models, has the cost structure been evaluated?",
+      "details": [
+        "Assessment of token-usage based pricing",
+        "Evaluation of cost variations based on initial prompt length and requested token response length",
+        "Analysis of cost differences across model versions",
+        "Examination of pricing structures for different types of requests",
+        "Consideration of volume discounts or tiered pricing models"
+      ]
+    },
+    {
+      "question": "Comprehensive Cost Tracking",
+      "explainer": "Does the evaluation attempt to track and account for both direct and indirect costs, including those not immediately tied to the system?",
+      "details": [
+        "Assessment of costs related to broader infrastructure or organizational changes",
+        "Evaluation of long-term maintenance and update costs",
+        "Analysis of costs associated with complementary technologies or processes",
+        "Consideration of costs related to regulatory compliance or legal considerations"
+      ]
+    }
+  ]
+}

scorecard_templates/privacy_data_protection.json

@@ -0,0 +1,91 @@
+{
+  "name": "Privacy and Data Protection Evaluation",
+  "questions": [
+    {
+      "question": "Data Minimization and Consent Practices",
+      "explainer": "Has the system been evaluated for its adherence to data minimization and consent practices?",
+      "details": [
+        "Implementation of data minimization practices",
+        "Use of opt-in data collection methods",
+        "Assessment of active consent for collecting, processing, and sharing data",
+        "Evaluation of compliance with privacy regulations (e.g., CCPA)",
+        "Measures for dataset transparency and accountability"
+      ]
+    },
+    {
+      "question": "Memorization and Data Leakage Evaluation",
+      "explainer": "Has the system been assessed for unintended memorization and data leakage?",
+      "details": [
+        "Examination of the maximum amount of discoverable information given training data",
+        "Evaluation of extractable information without training data access",
+        "Analysis of out-of-distribution data revelation",
+        "Assessment of factors increasing likelihood of memorization (e.g., parameter count, sample repetitions)",
+        "Use of Membership Inference Attacks (MIA) or similar techniques"
+      ]
+    },
+    {
+      "question": "Personal Information Revelation Assessment",
+      "explainer": "Has the system been evaluated for its potential to reveal personal or sensitive information?",
+      "details": [
+        "Direct prompting tests to reveal Personally Identifiable Information (PII)",
+        "Use of tools like ProPILE to audit PII revelation likelihood",
+        "Evaluation of the system's ability to infer personal attributes",
+        "Assessment of privacy violations based on Contextual Integrity and Theory of Mind",
+        "Analysis of the system's understanding of privacy context and purpose"
+      ]
+    },
+    {
+      "question": "Image and Audio Privacy Evaluation",
+      "explainer": "For image and audio generation systems, has privacy been evaluated?",
+      "details": [
+        "Assessment of training data memorization in image generation",
+        "Use of adversarial Membership Inference Attacks for images",
+        "Evaluation of the proportion of generated images with high similarity to training data",
+        "Detection of memorized prompts in image generation",
+        "Scrutiny of audio generation models' ability to synthesize particular individuals' audio"
+      ]
+    },
+    {
+      "question": "Intellectual Property and Copyright Evaluation",
+      "explainer": "Has the system been evaluated for its handling of intellectual property and copyrighted content?",
+      "details": [
+        "Assessment of the system's ability to generate copyrighted content",
+        "Evaluation of intellectual property concerns in generated content",
+        "Analysis of the system's handling of highly sensitive documents",
+        "Measures to prevent unauthorized use or reproduction of copyrighted material"
+      ]
+    },
+    {
+      "question": "Retroactive Privacy Protection",
+      "explainer": "Has the system been evaluated for its ability to implement retroactive privacy protections?",
+      "details": [
+        "Assessment of the system's capability to retroactively retrain in accordance with privacy policies",
+        "Evaluation of processes for removing specific data points upon request",
+        "Analysis of the system's adaptability to changing privacy regulations",
+        "Examination of the impact of data removal on model performance",
+        "Assessment of the timeframe and effectiveness of retroactive privacy measures"
+      ]
+    },
+    {
+      "question": "Third-party Hosting Privacy Evaluation",
+      "explainer": "For third-party hosted systems, has privacy been evaluated in the context of system prompts and hidden inputs?",
+      "details": [
+        "Assessment of potential leakage of private input data in generations",
+        "Evaluation of system prompt privacy, especially for prompts containing proprietary information",
+        "Analysis of the system's handling of sensitive database records in context learning",
+        "Examination of privacy measures for prepended system prompts",
+        "Assessment of the system's ability to maintain confidentiality of hidden inputs"
+      ]
+    },
+    {
+      "question": "Generative AI-Specific Privacy Measures",
+      "explainer": "Has the evaluation considered the challenges of applying traditional privacy protection methods to generative AI?",
+      "details": [
+        "Assessment of the applicability of data sanitization techniques to generative models",
+        "Evaluation of differential privacy approaches in the context of generative AI",
+        "Analysis of novel privacy protection methods designed specifically for generative models",
+        "Examination of the trade-offs between privacy protection and model performance in generative AI"
+      ]
+    }
+  ]
+}