Reduce memory usage

app.py

@@ -62,7 +62,8 @@ mapping_table_idx_dataset_idx = {}
 map_periodic_table = {v.symbol: k for k, v in enumerate(periodictable.elements)}
 n_elements = len(map_periodic_table)
 
-train_df = dataset.to_pandas()
+# Preprocessing step to create an index for the dataset
+train_df = dataset.select_columns(["chemical_formula_descriptive"]).to_pandas()
 
 pattern = re.compile(r"(?P<element>[A-Z][a-z]?)(?P<count>\d*)")
 extracted = train_df["chemical_formula_descriptive"].str.extractall(pattern)
@@ -79,6 +80,7 @@ wide_df = extracted.reset_index().pivot_table(  # Move index to columns for pivo
 all_elements = [el.symbol for el in periodictable.elements]  # full element list
 wide_df = wide_df.reindex(columns=all_elements, fill_value=0)
 
+
 dataset_index = wide_df.values
 
 dataset_index = dataset_index / np.sum(dataset_index, axis=1)[:, None]
@@ -86,6 +88,7 @@ dataset_index = (
     dataset_index / np.linalg.norm(dataset_index, axis=1)[:, None]
 )  # Normalize vectors
 
+del train_df, extracted, wide_df
 
 # Initialize the Dash app
 app = dash.Dash(__name__, assets_folder=SETTINGS.ASSETS_PATH)