\n'
for col, img_str in distribution_plots.items():
content += f"""
\n"""
content += "
\n\n"
# Add word clouds
if wordcloud_plots:
content += "### Word Clouds\n\n"
content += 'Distribution of {col}
\n'
for col, img_str in wordcloud_plots.items():
content += f"""
\n"""
content += "
\n"
return content
def generate_limitations_section(df: pd.DataFrame, analysis: dict) -> str:
"""Generate a section about dataset limitations and potential biases."""
limitations = [
"This dataset may not be representative of all possible scenarios or use cases.",
f"The dataset contains {len(df):,} records, which may limit its applicability to certain tasks.",
"There may be inherent biases in the data collection or annotation process.",
]
# Add warnings about missing values if present
missing_values = df.isnull().sum()
if missing_values.any():
limitations.append(
f"Some fields contain missing values: {', '.join(missing_values[missing_values > 0].index)}"
)
return f"""The following limitations and potential biases should be considered when using this dataset:
{chr(10).join(f'* {limitation}' for limitation in limitations)}
Please consider these limitations when using the dataset and validate results accordingly."""
def generate_usage_section(dataset_info: dict, analysis: dict) -> str:
"""Generate comprehensive usage guidelines."""
return f"""This dataset is released under the Apache 2.0 License. When using this dataset:
* 📚 Cite the dataset using the BibTeX entry provided below
* 🤝 Consider contributing improvements or reporting issues
* 💡 Share derivative works with the community when possible
* 🔍 Validate the dataset's suitability for your specific use case
* ⚠️ Be aware of the limitations and biases discussed above
* 📊 Consider the dataset size and computational requirements for your application
For questions or additional information, please visit the dataset repository on Hugging Face.
"""
def get_task_categories(df: pd.DataFrame, analysis: dict) -> list:
"""Infer potential task categories based on the data and analysis."""
categories = ["other"] # Default category
# Add more sophisticated task inference logic based on column names and content
text_columns = df.select_dtypes(include=["object"]).columns
numeric_columns = df.select_dtypes(include=["int64", "float64"]).columns
if len(text_columns) > 0:
categories.append("text-classification")
if len(numeric_columns) > 0:
categories.append("regression")
return list(set(categories)) # Remove duplicates
def clean_dataset_name(name: str) -> str:
"""Clean dataset name for citation."""
return name.replace("/", "_").replace("-", "_").lower()
def generate_schema_table(schema: dict) -> str:
"""Generate a markdown table for the schema, handling nested structures."""
# Table header
table = "| Field | Type | Description |\n| --- | --- | --- |\n"
# Generate rows recursively
rows = []
for field, info in schema.items():
rows.extend(format_schema_item(field, info))
# Join all rows
table += "\n".join(rows)
return table
def format_stats_section(stats: dict) -> str:
"""Format the statistics section of the dataset card."""
content = """### Basic Statistics
"""
# Add basic stats
for key, value in stats["basic_stats"].items():
# Convert key from snake_case to Title Case
formatted_key = key.replace("_", " ").title()
content += f"* {formatted_key}: {value}\n"
# Add token statistics if available
if stats.get("token_stats") and stats["token_stats"]["total"] > 0:
content += "\n### Token Statistics\n"
content += f"* Total Tokens: {stats['token_stats']['total']:,}\n"
if stats["token_stats"].get("by_column"):
content += "\n**Tokens by Column:**\n"
for col, count in stats["token_stats"]["by_column"].items():
content += f"* {col}: {count:,}\n"
return content
def format_schema_item(field_name: str, field_info: dict, prefix: str = "") -> list:
"""Recursively format schema items for nested structures."""
rows = []
# Handle nested objects
if isinstance(field_info, dict):
if "type" in field_info and "description" in field_info:
# This is a leaf node with type and description
rows.append(
f"| {prefix}{field_name} | {field_info['type']} | {field_info['description']} |"
)
else:
# This is a nested object, recurse through its properties
for subfield, subinfo in field_info.items():
if prefix:
new_prefix = f"{prefix}{field_name}."
else:
new_prefix = f"{field_name}."
rows.extend(format_schema_item(subfield, subinfo, new_prefix))
return rows
def flatten_list_column(data, column):
"""Flatten a column containing lists into individual values with counts."""
# Flatten the lists into individual items
flattened = [
item
for sublist in data[column]
if isinstance(sublist, list)
for item in sublist
]
# Count occurrences
value_counts = pd.Series(Counter(flattened))
return value_counts
Word Cloud for {col}