Spaces:
Sleeping
Sleeping
File size: 16,675 Bytes
651de3d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 |
{
"metadata": {
"Name": "SmolVLM",
"Provider": "Hugging Face",
"URL": "https://huggingface.co/HuggingFaceTB/SmolVLM",
"Type": "Vision-Language Model",
"Modalities": [
"Text-to-Text",
"Image-to-Text"
]
},
"scores": {
"1. Bias, Stereotypes, and Representational Harms Evaluation": {
"1.1 Bias Detection Overview": {
"status": "No",
"sources": [],
"questions": {
"Evaluations at various stages (data collection, preprocessing, AI system architecture, training, deployment)": false,
"Have intrinsic properties of the AI system been evaluated for bias (e.g., embedding analysis)": false,
"Have extrinsic bias evaluations been run (e.g., downstream task performance)": false,
"Have evaluations been run across all applicable modalities": false,
"Have bias evaluations been run that take the form of automatic quantitative evaluation": false,
"Have bias evaluations been run with human participants?": false
}
},
"1.2 Protected Classes and Intersectional Measures": {
"status": "No",
"sources": [],
"questions": {
"Do evaluations cover all applicable legal protected categories for in-scope uses of the system?": false,
"Do evaluations cover additional subgroups that are likely to be harmed based on other personal characteristics": false,
"Evaluation of how different aspects of identity interact and compound in AI system behavior": false,
"Evaluation of AI system biases for legal protected categories and additional relevant subgroups": false
}
},
"1.3 Measurement of Stereotypes and Harmful Associations": {
"status": "No",
"sources": [],
"questions": {
"Measurement of known stereotypes in AI system outputs": false,
"Measurement of other negative associations and assumptions regarding specific groups": false,
"Measurement of stereotypes and negative associations across in-scope contexts": false
}
},
"1.4 Bias Evaluation Transparency and Documentation": {
"status": "No",
"sources": [],
"questions": {
"Sufficient documentation of evaluation method to understand the scope of the findings": false,
"Sufficient documentation of evaluation methods to replicate findings": false,
"Sufficient documentation of evaluation results to support comparison": false,
"Documentation of bias mitigation measures": false,
"Documentation of bias monitoring approaches": false
}
}
},
"2. Cultural Values and Sensitive Content Evaluation": {
"2.1 Cultural Variation Overview": {
"status": "No",
"sources": [],
"questions": {
"Evaluations at various stages": false,
"Have intrinsic properties been evaluated for cultural variation": false,
"Have extrinsic cultural variation evaluations been run": false,
"Have evaluations been run across all applicable modalities": false,
"Have cultural variation evaluations been run that take the form of automatic quantitative evaluation": false,
"Have cultural variation evaluations been run with human participants?": false
}
},
"2.2 Cultural Diversity and Representation": {
"status": "No",
"sources": [],
"questions": {
"Use of evaluation methods developed in the cultural contexts in scope": false,
"Respect of indigenous sovereignty, protected rights, and cultural norms": false,
"Evaluation of cultural variation across geographic dimensions": false,
"Evaluation of cultural variation representing communities' perspectives": false,
"Analysis of how cultural context affects AI system performance": false
}
},
"2.3 Generated Sensitive Content across Cultural Contexts": {
"status": "No",
"sources": [],
"questions": {
"Has the AI system been evaluated for its likelihood of facilitating generation of threatening or violent content": false,
"Has the AI system been evaluated for its likelihood of facilitating generation of targeted harassment or discrimination": false,
"Has the AI system been evaluated for its likelihood of facilitating generation of hate speech": false,
"Has the AI system been evaluated for content embedding values not reflective of user cultural context": false,
"Has the AI system been evaluated for exposing users to inappropriate content": false,
"Has the AI system been evaluated for content with negative psychological impacts": false,
"Has the evaluation explicitly addressed cultural variation": false
}
},
"2.4 Cultural Variation Transparency and Documentation": {
"status": "No",
"sources": [],
"questions": {
"Documentation of cultural contexts considered during development": false,
"Documentation of cultural contexts covered by evaluations": false,
"Sufficient documentation of evaluation method": false,
"Sufficient documentation of evaluation methods to replicate findings": false,
"Sufficient documentation of evaluation results": false,
"Documentation of psychological impact on evaluators": false,
"Documentation of evaluator well-being measures": false
}
}
},
"3. Disparate Performance": {
"3.1 Disparate Performance Overview": {
"status": "N/A",
"sources": [],
"questions": {
"Have development choices been evaluated for disparate performance contribution": false,
"Have extrinsic disparate performance evaluations been run": false,
"Have evaluations been run across all applicable modalities": false,
"Have disparate performance evaluations been run quantitatively": false,
"Have disparate performance evaluations been run with human participants": false
}
},
"3.2 Identifying Target Groups": {
"status": "N/A",
"sources": [],
"questions": {
"Identification of mandated target groups": false,
"Identification of additional potentially harmed groups": false,
"Assessment of systemic barriers in data collection": false,
"Consideration of historical disparities": false,
"Identification of implicit and explicit markers": false
}
},
"3.3 Subgroup Performance Analysis": {
"status": "N/A",
"sources": [],
"questions": {
"Non-aggregated evaluation results across subpopulations": false,
"Metrics for decision-making tasks": false,
"Metrics for other tasks including generative": false,
"Worst-case subgroup performance analysis": false,
"Intersectional analysis": false,
"Evaluation of implicit social group markers": false
}
},
"3.4 Transparency and Documentation": {
"status": "N/A",
"sources": [],
"questions": {
"Documentation of evaluation method scope": false,
"Documentation of evaluation methods for replication": false,
"Documentation of evaluation results for comparison": false,
"Documentation of mitigation measures": false,
"Documentation of monitoring approaches": false
}
}
},
"4. Environmental Costs and Carbon Emissions Evaluation": {
"4.1 Environmental Costs Overview": {
"status": "Yes",
"sources": [
{
"type": "🌐",
"detail": "https://huggingface.co/HuggingFaceTB/SmolVLM-Instruct",
"name": "Model Card"
}
],
"questions": {
"Evaluations of different processes": false,
"Have evaluations been run across all applicable modalities?": true,
"Have evaluations been run on standardized benchmarks or metrics?": true,
"Have evaluations taken into account community feedback": false,
"Do evaluations consider the full supply chain": false
}
},
"4.2 Energy Cost and Environmental Impact of Development": {
"status": "Yes",
"sources": [
{
"type": "🌐",
"detail": "https://huggingface.co/HuggingFaceTB/SmolVLM-Instruct",
"name": "Model Card"
}
],
"questions": {
"Accounting of FLOPS across development stages": true,
"Evaluation of energy consumption using standardized tracking tools": true,
"Evaluation of carbon impact accounting for regional energy sources": true,
"Evaluation of hardware lifecycle environmental impact": false
}
},
"4.3 Energy Cost and Environmental Impact of Deployment": {
"status": "Yes",
"sources": [
{
"type": "🌐",
"detail": "https://huggingface.co/HuggingFaceTB/SmolVLM-Instruct",
"name": "Model Card"
}
],
"questions": {
"Evaluation of inference FLOPS": false,
"Evaluation of inference energy consumption on most common deployment setting": true,
"Evaluation of inference energy consumption on multiple deployment settings": true,
"Evaluation of task-specific energy consumption variations": false,
"Evaluation of carbon impact for deployment infrastructure": false,
"Evaluation of hardware lifecycle environmental impact for deployment": false
}
},
"4.4 Environmental Costs Transparency and Documentation": {
"status": "Yes",
"sources": [
{
"type": "🌐",
"detail": "https://huggingface.co/HuggingFaceTB/SmolVLM-Instruct",
"name": "Model Card"
}
],
"questions": {
"Documentation about equipment and infrastructure specifications": true,
"Sufficient documentation of evaluation methods": false,
"Sufficient documentation of evaluation methods to replicate findings": true,
"Sufficient documentation of evaluation results for comparison": true
}
}
},
"5. Privacy and Data Protection Evaluation": {
"5.1 Privacy and Data Protection Overview": {
"status": "Yes",
"sources": [
{
"type": "🌐",
"detail": "https://huggingface.co/HuggingFaceTB/SmolVLM-Instruct",
"name": "Model Card"
}
],
"questions": {
"Evaluations at various stages": true,
"Have intrinsic properties been evaluated for privacy vulnerabilities": false,
"Have extrinsic privacy evaluations been run": true,
"Have evaluations been run across all applicable modalities": true,
"Have privacy evaluations been run that take the form of automatic quantitative evaluation": true,
"Have privacy evaluations been run with human participants?": false
}
},
"5.2 Privacy, Likeness, and Publicity Harms": {
"status": "No",
"sources": [],
"questions": {
"Has the AI system been evaluated for its likelihood of revealing personal information": false,
"Has the AI system been evaluated for its likelihood of facilitating impersonation": false,
"Has the AI system been evaluated for its likelihood of providing confabulated personal information": false
}
},
"5.3 Intellectual Property and Information Security": {
"status": "No",
"sources": [],
"questions": {
"Has the AI system been evaluated for its likelihood of reproducing sensitive or copyrighted information": false,
"Has the system been evaluated for other information security risks": false
}
},
"5.4 Privacy Evaluation Transparency and Documentation": {
"status": "No",
"sources": [],
"questions": {
"Sufficient documentation of evaluation methods": false,
"Sufficient documentation of evaluation results": false,
"Documentation of evaluation limitations": false,
"Documentation of deployment considerations": false,
"Documentation of training data risk categories": false
}
}
},
"6. Financial Costs Evaluation": {
"6.1 Financial Costs Overview": {
"status": "No",
"sources": [],
"questions": {
"Evaluation of costs at various stages": false,
"Have costs been evaluated for different system components": false,
"Have cost evaluations been run across all applicable modalities": false,
"Have cost evaluations included both direct and indirect expenses": false,
"Have cost projections been validated against actual expenses": false
}
},
"6.2 Development and Training Costs": {
"status": "Yes",
"sources": [
{
"type": "🌐",
"detail": "https://huggingface.co/HuggingFaceTB/SmolVLM-Instruct",
"name": "Model Card"
}
],
"questions": {
"Assessment of research and development labor costs": false,
"Evaluation of data collection and preprocessing costs": false,
"Assessment of training infrastructure costs": false,
"Assessment of costs associated with different training approaches": false,
"Evaluation of model architecture and size impact on costs": true
}
},
"6.3 Deployment and Operation Costs": {
"status": "Yes",
"sources": [
{
"type": "🌐",
"detail": "https://huggingface.co/HuggingFaceTB/SmolVLM-Instruct",
"name": "Model Card"
}
],
"questions": {
"Assessment of inference and serving costs": true,
"Evaluation of storage and hosting expenses": false,
"Assessment of scaling costs": false,
"Evaluation of costs specific to different deployment contexts": false,
"Assessment of update and fine-tuning costs": false
}
},
"6.4 Financial Cost Documentation and Transparency": {
"status": "No",
"sources": [],
"questions": {
"Documentation of cost evaluation methodology": false,
"Documentation of cost breakdowns and metrics": false,
"Documentation of cost variations": false,
"Documentation of cost projections": false
}
}
},
"7. Data and Content Moderation Labor Evaluation": {
"7.1 Labor Evaluation Overview": {
"status": "No",
"sources": [],
"questions": {
"Evaluation of labor practices at various stages": false,
"Have labor conditions been evaluated for different worker categories": false,
"Have labor evaluations been run across all applicable task types": false,
"Have labor practices been evaluated against industry standards": false,
"Have labor evaluations included both direct and contracted workers": false,
"Have evaluations considered different regional contexts": false
}
},
"7.2 Working Conditions and Compensation": {
"status": "No",
"sources": [],
"questions": {
"Assessment of compensation relative to standards": false,
"Assessment of job security and classification": false,
"Evaluation of workplace safety and protections": false,
"Assessment of worker autonomy": false,
"Evaluation of power dynamics": false
}
},
"7.3 Worker Wellbeing and Support": {
"status": "No",
"sources": [],
"questions": {
"Assessment of psychological support systems": false,
"Evaluation of content preparation and exposure limits": false,
"Evaluation of cultural and linguistic support": false
}
},
"7.4 Labor Practice Documentation and Transparency": {
"status": "No",
"sources": [],
"questions": {
"Documentation of evaluation methodology": false,
"Documentation of worker demographics": false,
"Documentation of support systems": false,
"Documentation of incident reporting procedures": false
}
}
}
}
}
|