scorecard_templates/environmental_costs.json

{
  "name": "Environmental Costs and Carbon Emissions Evaluation",
  "questions": [
    {
      "question": "Energy Consumption Measurement",
      "explainer": "Has the energy consumption of the system been measured across its lifecycle?",
      "details": [
        "Measurement of energy used in training, testing, and deploying the system",
        "Evaluation of compute power consumption",
        "Assessment of energy resources used by large-scale systems",
        "Tracking of energy usage across different stages of development"
      ]
    },
    {
      "question": "Carbon Footprint Quantification",
      "explainer": "Has the carbon footprint of the system been quantified?",
      "details": [
        "Use of tools like CodeCarbon or Carbontracker",
        "Measurement of carbon emissions for training and inference",
        "Conversion of energy consumption to carbon emissions",
        "Consideration of regional variations in energy sources and carbon intensity"
      ]
    },
    {
      "question": "Hardware Resource Evaluation",
      "explainer": "Has the system been evaluated for its use of hardware resources?",
      "details": [
        "Assessment of CPU, GPU, and TPU usage",
        "Measurement of FLOPS (Floating Point Operations)",
        "Evaluation of package power draw and GPU performance state",
        "Analysis of memory usage"
      ]
    },
    {
      "question": "Comprehensive Environmental Impact Assessment",
      "explainer": "Has a holistic evaluation of the system's environmental impact been conducted?",
      "details": [
        "Use of Life Cycle Assessment (LCA) methodologies",
        "Consideration of supply chains and manufacturing impacts",
        "Evaluation of immediate impacts of applying ML",
        "Assessment of system-level environmental impacts"
      ]
    },
    {
      "question": "Transparency in Environmental Reporting",
      "explainer": "Is there transparency in reporting the environmental costs and limitations of the evaluation?",
      "details": [
        "Disclosure of uncertainty around measured variables",
        "Reporting of marginal costs (e.g., added parameters' contribution to energy consumption)",
        "Transparency about equipment manufacturers and data/hosting centers",
        "Acknowledgment of limitations in accurately estimating GPU footprints and hosting-side impacts"
      ]
    },
    {
      "question": "Comprehensive Environmental Impact Metrics",
      "explainer": "Does the evaluation acknowledge the lack of consensus on environmental impact metrics and attempt to use comprehensive measures?",
      "details": [
        "Discussion of different approaches to measuring environmental impact",
        "Use of diverse measurements beyond energy consumption",
        "Consideration of various factors including lifecycle assessment",
        "Transparency about chosen metrics and their limitations"
      ]
    }
  ]
}