Deploy (see actual commits on https://github.com/mlcommons/croissant).

app.py

@@ -21,10 +21,10 @@ init_state()
 user = get_user()
 
 if OAUTH_CLIENT_ID and not user:
-    query_params = st.experimental_get_query_params()
-    state = query_params.get("state")
+    query_params = st.query_params
+    state = query_params.get_all("state")
     if state and state[0] == OAUTH_STATE:
-        code = query_params.get("code")
+        code = query_params["code"]
         if not code:
             st.stop()
         try:
@@ -34,7 +34,7 @@ if OAUTH_CLIENT_ID and not user:
         except:
             raise
         finally:
-            st.experimental_set_query_params()
+            st.query_params.clear()
     else:
         redirect_uri = urllib.parse.quote(REDIRECT_URI, safe="")
         client_id = urllib.parse.quote(OAUTH_CLIENT_ID, safe="")
@@ -48,7 +48,7 @@ if OAUTH_CLIENT_ID and not user:
 
 def _back_to_menu():
     """Sends the user back to the menu."""
-    st.experimental_set_query_params()
+    st.query_params.clear()
     init_state(force=True)
 
 

core/query_params.py

@@ -15,28 +15,24 @@ class QueryParams:
     OPEN_RECORD_SET = "recordSet"
 
 
-def _get_query_param(params: dict[str, Any], name: str) -> str | None:
+def _get_query_param(name: str) -> str | None:
     """Gets query param with the name `name`."""
-    if name in params:
-        param = params[name]
-        if isinstance(param, list) and len(param) > 0:
-            return param[0]
+    param = st.query_params.get_all(name)
+    if isinstance(param, list) and len(param) > 0:
+        return param[0]
     return None
 
 
 def _set_query_param(param: str, new_value: str) -> str | None:
-    params = st.experimental_get_query_params()
-    if params.get(param) == [new_value]:
+    params = st.query_params
+    if params.get_all(param) == [new_value]:
         # The value already exists in the query params.
         return
-    new_params = {k: v for k, v in params.items() if k != param}
-    new_params[param] = new_value
-    st.experimental_set_query_params(**new_params)
+    params[param] = new_value
 
 
 def is_record_set_expanded(record_set: RecordSet) -> bool:
-    params = st.experimental_get_query_params()
-    open_record_set_name = _get_query_param(params, QueryParams.OPEN_RECORD_SET)
+    open_record_set_name = _get_query_param(QueryParams.OPEN_RECORD_SET)
     if open_record_set_name:
         return open_record_set_name == record_set.name
     return False
@@ -47,8 +43,7 @@ def expand_record_set(record_set: RecordSet) -> None:
 
 
 def get_project_timestamp() -> str | None:
-    params = st.experimental_get_query_params()
-    return _get_query_param(params, QueryParams.OPEN_PROJECT)
+    return _get_query_param(QueryParams.OPEN_PROJECT)
 
 
 def set_project(project: CurrentProject):

core/state.py

@@ -127,6 +127,7 @@ class SelectedRecordSet:
 class FileObject:
     """FileObject analogue for editor"""
 
+    ctx: mlc.Context = dataclasses.field(default_factory=mlc.Context)
     name: str | None = None
     description: str | None = None
     contained_in: list[str] | None = dataclasses.field(default_factory=list)
@@ -135,7 +136,6 @@ class FileObject:
     encoding_format: str | None = None
     sha256: str | None = None
     df: pd.DataFrame | None = None
-    rdf: mlc.Rdf = dataclasses.field(default_factory=mlc.Rdf)
     folder: epath.PathLike | None = None
 
 
@@ -143,23 +143,23 @@ class FileObject:
 class FileSet:
     """FileSet analogue for editor"""
 
+    ctx: mlc.Context = dataclasses.field(default_factory=mlc.Context)
     contained_in: list[str] = dataclasses.field(default_factory=list)
     description: str | None = None
     encoding_format: str | None = ""
     includes: str | None = ""
     name: str = ""
-    rdf: mlc.Rdf = dataclasses.field(default_factory=mlc.Rdf)
 
 
 @dataclasses.dataclass
 class Field:
     """Field analogue for editor"""
 
+    ctx: mlc.Context = dataclasses.field(default_factory=mlc.Context)
     name: str | None = None
     description: str | None = None
     data_types: str | list[str] | None = None
     source: mlc.Source | None = None
-    rdf: mlc.Rdf = dataclasses.field(default_factory=mlc.Rdf)
     references: mlc.Source | None = None
 
 
@@ -167,13 +167,13 @@ class Field:
 class RecordSet:
     """Record Set analogue for editor"""
 
+    ctx: mlc.Context = dataclasses.field(default_factory=mlc.Context)
     name: str = ""
     data: list[Any] | None = None
     description: str | None = None
     is_enumeration: bool | None = None
     key: str | list[str] | None = None
     fields: list[Field] = dataclasses.field(default_factory=list)
-    rdf: mlc.Rdf = dataclasses.field(default_factory=mlc.Rdf)
 
 
 @dataclasses.dataclass
@@ -182,9 +182,10 @@ class Metadata:
 
     name: str = ""
     description: str | None = None
-    citation: str | None = None
-    conforms_to: str | None = None
+    cite_as: str | None = None
+    context: dict = dataclasses.field(default_factory=dict)
     creators: list[mlc.PersonOrOrganization] = dataclasses.field(default_factory=list)
+    ctx: mlc.Context = dataclasses.field(default_factory=mlc.Context)
     data_biases: str | None = None
     data_collection: str | None = None
     date_published: datetime.datetime | None = None
@@ -193,7 +194,6 @@ class Metadata:
     url: str = ""
     distribution: list[FileObject | FileSet] = dataclasses.field(default_factory=list)
     record_sets: list[RecordSet] = dataclasses.field(default_factory=list)
-    rdf: mlc.Rdf = dataclasses.field(default_factory=mlc.Rdf)
     version: str | None = None
 
     def __bool__(self):
@@ -295,17 +295,20 @@ class Metadata:
 
     def to_canonical(self) -> mlc.Metadata:
         distribution = []
+        ctx = self.ctx
         for file in self.distribution:
             if isinstance(file, FileObject):
-                distribution.append(create_class(mlc.FileObject, file))
+                distribution.append(create_class(mlc.FileObject, file, ctx=ctx))
             elif isinstance(file, FileSet):
-                distribution.append(create_class(mlc.FileSet, file))
+                distribution.append(create_class(mlc.FileSet, file, ctx=ctx))
         record_sets = []
         for record_set in self.record_sets:
             fields = []
             for field in record_set.fields:
-                fields.append(create_class(mlc.Field, field))
-            record_sets.append(create_class(mlc.RecordSet, record_set, fields=fields))
+                fields.append(create_class(mlc.Field, field, ctx=ctx))
+            record_sets.append(
+                create_class(mlc.RecordSet, record_set, ctx=ctx, fields=fields)
+            )
         return create_class(
             mlc.Metadata,
             self,

cypress.config.js

@@ -3,6 +3,6 @@ const { defineConfig } = require("cypress");
 module.exports = defineConfig({
   // To access content within Streamlit iframes for custom components:
   chromeWebSecurity: false,
-  defaultCommandTimeout: 10000,
+  defaultCommandTimeout: 20000,
   e2e: {},
 });

events/metadata.py

@@ -92,12 +92,11 @@ class MetadataEvent(enum.Enum):
     """Event that triggers a metadata change."""
 
     NAME = "NAME"
-    CONFORMS_TO = "CONFORMS_TO"
     DESCRIPTION = "DESCRIPTION"
     DATE_PUBLISHED = "DATE_PUBLISHED"
     URL = "URL"
     LICENSE = "LICENSE"
-    CITATION = "CITATION"
+    CITE_AS = "CITE_AS"
     VERSION = "VERSION"
     DATA_BIASES = "DATA_BIASES"
     DATA_COLLECTION = "DATA_COLLECTION"
@@ -111,14 +110,12 @@ class MetadataEvent(enum.Enum):
 def handle_metadata_change(event: MetadataEvent, metadata: Metadata, key: str):
     if event == MetadataEvent.NAME:
         metadata.name = find_unique_name(set(), st.session_state[key])
-    if event == MetadataEvent.CONFORMS_TO:
-        metadata.conforms_to = st.session_state[key]
     elif event == MetadataEvent.DESCRIPTION:
         metadata.description = st.session_state[key]
     elif event == MetadataEvent.LICENSE:
         metadata.license = LICENSES.get(st.session_state[key])
-    elif event == MetadataEvent.CITATION:
-        metadata.citation = st.session_state[key]
+    elif event == MetadataEvent.CITE_AS:
+        metadata.cite_as = st.session_state[key]
     elif event == MetadataEvent.URL:
         metadata.url = st.session_state[key]
     elif event == MetadataEvent.VERSION:

events/resources.py

@@ -72,6 +72,6 @@ def _create_instance1_from_instance2(instance1: Resource, instance2: type):
     attributes1 = set((field.name for field in dataclasses.fields(instance1)))
     attributes2 = set((field.name for field in dataclasses.fields(instance2)))
     common_attributes = attributes2.intersection(attributes1)
-    return instance2(**{
-        attribute: getattr(instance1, attribute) for attribute in common_attributes
-    })
+    return instance2(
+        **{attribute: getattr(instance1, attribute) for attribute in common_attributes}
+    )

views/jsonld.py

@@ -47,7 +47,7 @@ def render_jsonld():
         if croissant.metadata:
             metadata = mlc.Metadata(
                 name=croissant.metadata.name,
-                citation=croissant.metadata.citation,
+                cite_as=croissant.metadata.cite_as,
                 license=croissant.metadata.license,
                 description=croissant.metadata.description,
                 url=croissant.metadata.url,

views/metadata.py

@@ -97,14 +97,14 @@ def _render_generic_metadata(metadata: Metadata):
         on_change=handle_metadata_change,
         args=(MetadataEvent.LICENSE, metadata, key),
     )
-    key = "metadata-citation"
+    key = "metadata-cite-as"
     st.text_area(
         label="Citation",
         key=key,
-        value=metadata.citation,
+        value=metadata.cite_as,
         placeholder="@book{\n  title={Title}\n}",
         on_change=handle_metadata_change,
-        args=(MetadataEvent.CITATION, metadata, key),
+        args=(MetadataEvent.CITE_AS, metadata, key),
     )
     key = "metadata-date-published"
     st.date_input(

views/overview.py

@@ -10,7 +10,7 @@ from utils import needed_field
 from views.metadata import handle_metadata_change
 from views.metadata import MetadataEvent
 
-_NON_RELEVANT_METADATA = ["name", "distribution", "record_sets", "rdf"]
+_NON_RELEVANT_METADATA = ["ctx", "name", "distribution", "record_sets"]
 
 _INFO_TEXT = """Croissant files are composed of three layers:
 
@@ -98,7 +98,7 @@ def render_overview():
         if user_started_editing:
             warning = ""
             try:
-                issues = metadata.to_canonical().issues
+                issues = metadata.to_canonical().ctx.issues
                 if issues.errors:
                     warning += "**Errors**\n"
                     for error in issues.errors:

views/record_sets.py

@@ -44,11 +44,9 @@ class _Result(TypedDict):
 @st.cache_data(
     show_spinner="Generating the dataset...",
     hash_funcs={
-        "mlcroissant.Metadata": hash,
-        "mlcroissant.Field": hash,
-        "mlcroissant.FileObject": hash,
-        "mlcroissant.FileSet": hash,
-        "mlcroissant.RecordSet": hash,
+        "core.state.RecordSet": lambda record_set: hash(
+            (record_set.name, record_set.description)
+        ),
     },
 )
 def _generate_data_with_timeout(record_set: RecordSet) -> _Result:
@@ -385,11 +383,13 @@ def _render_left_panel():
                         "⚠️",
                         key=f"idea-{prefix}",
                         on_click=lambda: _generate_data_with_timeout.clear(),
-                        help=textwrap.dedent(f"""**Error**:
+                        help=textwrap.dedent(
+                            f"""**Error**:
 ```
 {exception}
 ```
-"""),
+"""
+                        ),
                     )
                 right.markdown("No preview is possible.")
 

views/record_sets_test.py

@@ -19,10 +19,12 @@ def test_find_joins():
             references=mlc.Source(uid="some_other_record_set/some_other_field"),
         ),
     ]
-    assert _find_joins(fields) == set([
-        (("some_csv", "some_column"), ("some_record_set", "some_field")),
-        (
-            ("some_record_set", "some_field"),
-            ("some_other_record_set", "some_other_field"),
-        ),
-    ])
+    assert _find_joins(fields) == set(
+        [
+            (("some_csv", "some_column"), ("some_record_set", "some_field")),
+            (
+                ("some_record_set", "some_field"),
+                ("some_other_record_set", "some_other_field"),
+            ),
+        ]
+    )

views/splash.py

@@ -65,11 +65,11 @@ def render_splash():
         with st.expander("**Load an existing dataset**", expanded=True):
 
             def create_example(dataset: str):
-                base = f"https://raw.githubusercontent.com/mlcommons/croissant/main/datasets/{dataset.lower()}"
+                base = f"https://raw.githubusercontent.com/mlcommons/croissant/main/datasets/1.0/{dataset.lower()}"
                 url = f"{base}/metadata.json"
                 try:
                     json = requests.get(url).json()
-                    metadata = mlc.Metadata.from_json(mlc.Issues(), json, None)
+                    metadata = mlc.Metadata.from_json(mlc.Context(), json)
                     st.session_state[Metadata] = Metadata.from_canonical(metadata)
                     save_current_project()
                     # Write supplementary files.