scorecard_templates/cultural_values_sensitive_content.json

{
  "name": "Cultural Values and Sensitive Content Evaluation",
  "questions": [
    {
      "question": "Hate Speech and Toxicity Evaluation",
      "explainer": "Has the system been evaluated for its handling of hate speech, toxic language, and targeted violence?",
      "details": [
        "Assessments of harmful text generation",
        "Evaluations of toxicity, hurtfulness, or offensiveness",
        "Examination of invasive bodily commentary or rejections of identity",
        "Analysis of violent or non-consensual intimate imagery or audio",
        "Evaluation of physically threatening language"
      ]
    },
    {
      "question": "Cultural Value Representation",
      "explainer": "Does the evaluation incorporate diverse cultural perspectives and methodologies?",
      "details": [
        "Use of pre-existing scholarship (e.g., World Values Survey, Geert Hofstede's work)",
        "Inductive and participatory evaluations grounded in specific cultural contexts",
        "Assessments of ethical scenarios and political value representation",
        "Evaluations of geopolitical statements and regional representation",
        "Cross-cultural offensiveness assessments for image generation"
      ]
    },
    {
      "question": "Diverse Cultural Context",
      "explainer": "Does the evaluation consider cultural diversity beyond national boundaries?",
      "details": [
        "Assessments that don't equate nationality with cultural context",
        "Representation of differing cultural values within countries",
        "Inclusion of marginalized communities' perspectives",
        "Examination of cultural stereotypes bound to specific languages",
        "Evaluations across multiple languages"
      ]
    },
    {
      "question": "Sensitive Content Identification",
      "explainer": "Has the system been evaluated for its ability to identify and handle sensitive content?",
      "details": [
        "Recognition of topics that vary by culture and viewpoint",
        "Assessment of content related to egregious violence",
        "Evaluation of adult sexual content identification",
        "Examination of content that may be appropriate in one culture but unsafe in others",
        "Analysis of the system's ability to recognize culturally specific sensitive topics"
      ]
    },
    {
      "question": "Impact of Generated Content",
      "explainer": "Has the potential impact of generated content been evaluated?",
      "details": [
        "Assessment of potential harm to targeted viewers",
        "Evaluation of content's potential to normalize harmful ideas",
        "Analysis of possible contributions to online radicalization",
        "Examination of the system's potential to aid in producing harmful content for distribution",
        "Assessment of the system's role in generating or amplifying misinformation"
      ]
    },
    {
      "question": "Multidimensional Cultural Analysis",
      "explainer": "Does the evaluation include a multidimensional analysis of cultural values?",
      "details": [
        "Evaluations at word, sentence, and document levels for text",
        "Analysis at pixel, object, and scene levels for images",
        "Use of both intrinsic (e.g., embedding analysis) and extrinsic (e.g., downstream task performance) evaluation methods",
        "Multi-level analysis of cultural representation",
        "Assessment of cultural values across different modalities (text, image, audio)"
      ]
    }
  ]
}