|
import streamlit as st |
|
import os |
|
import subprocess |
|
import sys |
|
import shutil |
|
from Crypto.PublicKey import RSA |
|
from datasets import load_dataset |
|
import pandas as pd |
|
import oci |
|
from oci import object_storage |
|
from oci.object_storage.models import CreateBucketDetails |
|
from oci.object_storage.models import CreatePreauthenticatedRequestDetails |
|
import pickle |
|
from PIL import Image |
|
|
|
|
|
|
|
st.set_page_config(page_title="Oracle") |
|
image = Image.open('oracle_huggingface.png') |
|
st.image(image) |
|
|
|
st.title("Oracle x HF") |
|
st.header("Upload HF Dataset to OCI Object Storage!") |
|
st.caption("The first of many features and integrations between HF and OCI") |
|
|
|
|
|
config_location = ".oci/config" |
|
user_ocid = "" |
|
tenancy_ocid = "" |
|
region = "" |
|
gen_api_key = "n" |
|
private_key_location = ".oci/private_key.pem" |
|
hf_dataset = "" |
|
|
|
|
|
|
|
|
|
|
|
oracle_form = st.form("configuration") |
|
oracle_form.write("OCI Settings") |
|
user_ocid = oracle_form.text_input("Enter the User OCID", help="To find your user OCID, first login to your OCI account. From the home screen, click on the top right face icon marked as profile and click on your email. From here, find and copy your OCID into this box") |
|
tenancy_ocid = oracle_form.text_input("Enter the tenancy ocid", help="To find your tenancy OCID, first login to your OCI account. From the home screen, click on the top right face icon marked as profile and click on your tenancy. From here, find and copy your OCID into this box") |
|
region = oracle_form.text_input("Enter the region", "us-ashburn-1", help="To find your region identifier, first login to your OCI account. From the home screen, click on the top right region name. Scroll to the bottom of the regions list to the option labeled as manage region. Within this, the list of region identifiers is available. Copy your corresponding region identifier into here.") |
|
existing_checkbox = oracle_form.checkbox("Check this if you want to put the dataset into an existing bucket rather than create a new one") |
|
|
|
|
|
|
|
oracle_submitted = oracle_form.form_submit_button("Generate API Key") |
|
oracle_form.write("After the API Key is generated, go to the same screen as where you got your user_ocid from. Scroll down to the bottom left and look for the API Key section under the Resources tab . Click on this, and then click the Add API Key button. Click on the Paste Public Key option. Copy and Paste the Generated API Key") |
|
|
|
|
|
|
|
dataset_form = st.form("dataset") |
|
dataset_form.write("Dataset Settings") |
|
dataset_name = dataset_form.text_area("Enter the name of the huggingface Dataset:", value = "biosses", help ="Datasets can be found here: https://huggingface.co/datasets The default, BIOSSES is a benchmark dataset for biomedical sentence similarity estimation.") |
|
dataset_name_2 = dataset_form.text_area("Enter the name of the config for the dataset if it has one", value = " ", help="Some datasets have config options associated with them, enter them here") |
|
split_name = dataset_form.text_area("Enter the name of the split of the dataset that you want to use", value = "train") |
|
pd_checkbox = dataset_form.checkbox("Check this if you want this to be uploaded as a pandas dataframe instead of a HF Dataset Object") |
|
dataset_submitted = dataset_form.form_submit_button("Upload Dataset to OCI Object Storage") |
|
|
|
|
|
def load_and_process_data(path, name, streaming, split_name): |
|
dataset = load_dataset(path = path, name = name, streaming=streaming, keep_in_memory = True) |
|
|
|
dataset_head = dataset[split_name] |
|
return dataset_head |
|
|
|
|
|
|
|
if oracle_submitted: |
|
input_str = config_location + "\n" + "Y" + "\n" + "USER" + "\n" + user_ocid + "\n" + tenancy_ocid + "\n" + region + "\n" + gen_api_key + "\n" + private_key_location |
|
|
|
key_input_str = " \n" + " \n" |
|
|
|
try: |
|
shutil.rmtree(".oci") |
|
except Exception: |
|
pass |
|
try: |
|
os.mkdir(".oci") |
|
except FileExistsError: |
|
pass |
|
|
|
open(".oci/config", "a").close() |
|
|
|
|
|
key = RSA.generate(2048) |
|
private_key = key.export_key() |
|
file_out = open(".oci/private_key.pem", "wb") |
|
file_out.write(private_key) |
|
file_out.close() |
|
|
|
public_key = key.publickey().export_key() |
|
file_out = open(".oci/public_key.pem", "wb") |
|
file_out.write(public_key) |
|
file_out.close() |
|
|
|
p = subprocess.run(["oci", "setup", "config"], text = True, input = input_str) |
|
|
|
cat_public = subprocess.run(["cat", ".oci/public_key.pem"], text = True, capture_output=True) |
|
cat_config = subprocess.run(["cat", ".oci/config"], text = True, capture_output=True) |
|
oracle_form.text(cat_public.stdout) |
|
with oracle_form.expander("Open to see the generated OCI config file"): |
|
oracle_form.text(cat_config.stdout) |
|
|
|
|
|
|
|
if dataset_submitted: |
|
hf_dataset = load_and_process_data(dataset_name, dataset_name_2, False, split_name) |
|
if pd_checkbox: |
|
hf_dataset = pd.DataFrame.from_dict(hf_dataset) |
|
st.write(hf_dataset) |
|
st.write("Dataset Pulled Succesfully!") |
|
oci_config = oci.config.from_file(".oci/config", profile_name = "USER") |
|
object_storage = object_storage.ObjectStorageClient(oci_config) |
|
st.write("Object Storage Connected Succesfully") |
|
namespace = object_storage.get_namespace().data |
|
compartment_id = oci_config["tenancy"] |
|
st.write(namespace) |
|
bucket_name = dataset_name.replace("/", "-") |
|
try: |
|
bucket = object_storage.create_bucket( |
|
namespace, |
|
oci.object_storage.models.CreateBucketDetails( |
|
name=bucket_name, |
|
compartment_id=compartment_id, |
|
storage_tier='Archive', |
|
public_access_type='ObjectRead' |
|
) |
|
) |
|
st.write("Bucket Written:") |
|
st.write(bucket) |
|
except Exception: |
|
st.write("Bucket Exists, Writing Dataset to Bucket") |
|
|
|
st.write("Uploading new object if it doesn't already exist {!r}".format(hf_dataset)) |
|
hf_bytes = pickle.dumps(hf_dataset) |
|
obj = object_storage.put_object( |
|
namespace, |
|
bucket_name, |
|
bucket_name, |
|
hf_bytes) |
|
st.write("Object Uploaded Successfully!") |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
