Spaces:
Sleeping
Sleeping
File size: 2,922 Bytes
221451e |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 |
import streamlit as st
import boto3
from botocore.exceptions import NoCredentialsError, ClientError, EndpointConnectionError
from collections import defaultdict
from datetime import datetime
# AWS S3 setup
os.environ['AWS_ACCESS_KEY']
os.environ['AWS_SECRET_KEY']
os.environ['BUCKET_NAME']
# Initialize a session using DigitalOcean Spaces
session = boto3.session.Session()
s3 = session.client('s3',
region_name='us-west-2', # Update to the correct region
aws_access_key_id=AWS_ACCESS_KEY,
aws_secret_access_key=AWS_SECRET_KEY)
def list_folders(bucket_name):
try:
folders = []
result = s3.list_objects_v2(Bucket=bucket_name, Delimiter='/')
for o in result.get('CommonPrefixes', []):
folders.append(o.get('Prefix'))
return folders
except NoCredentialsError:
st.error("Credentials not available")
return []
except ClientError as e:
st.error(f"Client error: {e}")
return []
except EndpointConnectionError as e:
st.error(f"Endpoint connection error: {e}")
return []
def count_files_in_folder(bucket_name, prefix):
try:
folder_counts = defaultdict(lambda: defaultdict(int))
paginator = s3.get_paginator('list_objects_v2')
pages = paginator.paginate(Bucket=bucket_name, Prefix=prefix)
for page in pages:
for obj in page.get('Contents', []):
folder = '/'.join(obj['Key'].split('/')[:-1])
last_modified = obj['LastModified'].strftime('%Y-%m-%d')
folder_counts[folder][last_modified] += 1
return folder_counts
except NoCredentialsError:
st.error("Credentials not available")
return {}
except ClientError as e:
st.error(f"Client error: {e}")
return {}
except EndpointConnectionError as e:
st.error(f"Endpoint connection error: {e}")
return {}
# Streamlit UI
st.title('NeuroSinQ IAT Completions Dashboard')
# Dropdown to select folder
folders = list_folders(BUCKET_NAME)
#selected_folder = st.selectbox('Select a folder', folders)
selected_folder = st.selectbox('Select a folder', folders, disabled=True, index=folders.index('JUNIPER/') if 'JUNIPER/' in folders else 0)
# Display file counts
if selected_folder:
folder_counts = count_files_in_folder(BUCKET_NAME, selected_folder)
if folder_counts:
# Collect all dates
all_dates = sorted(set(date for counts in folder_counts.values() for date in counts))
# Prepare data for display
data = []
for folder, counts in folder_counts.items():
row = {'Folder': folder}
for date in all_dates:
row[date] = counts.get(date, 0)
data.append(row)
st.write("File counts by folder and last modified date:")
st.table(data)
|