import functions as funky import pandas as pd import gradio as gr import os from datasets import load_dataset from huggingface_hub import login import numpy as np from fastapi import FastAPI, Request import uvicorn from starlette.middleware.sessions import SessionMiddleware import fastapi login(token = os.environ['HUB_TOKEN']) logger = gr.HuggingFaceDatasetSaver(os.environ['HUB_TOKEN'], dataset_name='illustration_gdrive_logging', organization=None, private=True) logger.setup([gr.Text(label="clicked_url"), gr.Text(label="seach_term"), gr.Text(label = 'sessionhash')], './flagged_data_points') logging_js = ''' function magicFunc(x){ let script = document.createElement('script'); script.innerHTML = "async function magicFunc(x){let z = document.getElementById('search_term').getElementsByTagName('textarea')[0].value; await fetch('/track?url=' + x + '&q=' + z)}"; document.head.appendChild(script); } ''' dataset = load_dataset("bradley6597/illustration-test") df = pd.DataFrame(dataset['train']).drop_duplicates() ill_links = df.copy() ill_links = ill_links[ill_links['Description'] != 'Moved'].copy() ill_links['code'] = ill_links['link'].str.replace("https://drive.google.com/file/d/", "", regex = False) ill_links['code'] = ill_links['code'].str.replace("/view?usp=drivesdk", "", regex = False) # ill_links['image_code'] = 'https://lh3.google.com/u/0/d/' + ill_links['code'] + '=k' ill_links['image_code'] = 'https://lh3.google.com/u/0/d/' + ill_links['code'] + '=w320-h304' ill_links['image_code'] = '
{i}
' + tmp_links ill_check_lst.append(tmp_links) ill_check_df = pd.DataFrame(ill_check_lst).T ill_check_html = ill_check_df.to_html(escape = False, render_links = True, index = False, header = False) ind_main, doc_main, tf_main = funky.index_documents(ill_links) ind_title, doc_title, tf_title = funky.index_documents(ill_links_title) def same_auth(username, password): return(username == os.environ['username']) & (password == os.environ['password']) def search_index(search_text, sd, ks, sort_by, max_results, user_num, search_title): if search_title: output = funky.search(tf_title, doc_title, ind_title, search_text, search_type = 'AND', ranking = True) else: output = funky.search(tf_main, doc_main, ind_main, search_text, search_type='AND', ranking = True) output = [x for o in output for x in o if type(x) is not float] output_df = pd.DataFrame(output).reset_index(drop = True) if output_df.shape[0] > 0: output_df['url'] = output_df['url'].str.replace("/u/0/", f"/u/{int(user_num)}/", regex = False) if len(sd) == 1: output_df = output_df[(output_df['filepath'].str.contains(str(sd[0]), regex = False))] if len(ks) > 0: keystage_filter = '|'.join(ks).lower() if search_title: output_df['abstract'] = output_df['abstract'] + ' ' + output_df['Description'] output_df['abstract'] = output_df['abstract'].str.lower() output_df['post_filepath'] = output_df['post_filepath'].str.lower() output_df['missing_desc'] = np.where(output_df['abstract'].str.contains('eyfs|ks1|ks2', regex = True), 0, 1) output_df2 = output_df[(output_df['abstract'].str.contains(keystage_filter, regex = True) | (output_df['missing_desc'] == 1))].copy() output_df2 = output_df2[(output_df2['post_filepath'].str.contains(keystage_filter, regex = True))] if output_df2.shape[0] == 0: output_df2 = output_df[(output_df['post_filepath'].str.contains(keystage_filter, regex = True))] output_df2['ind'] = output_df2.index if sort_by == 'Relevance': output_df2 = output_df2.sort_values(by = ['missing_desc', 'ind'], ascending = [True, True]) elif sort_by == 'Date Created': output_df2 = output_df2.sort_values(by = ['Date Created'], ascending = False) elif sort_by == 'A-Z': output_df2 = output_df2.sort_values(by = ['title'], ascending = True) output_df2 = output_df2.head(int(max_results)) output_df2 = output_df2[['url']].reset_index(drop = True) max_cols = 5 output_df2['row'] = output_df2.index % max_cols for x in range(0, max_cols): tmp = output_df2[output_df2['row'] == x].reset_index(drop = True) tmp = tmp[['url']] if x == 0: final_df = tmp else: final_df = pd.concat([final_df, tmp], axis = 1) final_df = final_df.fillna('') else: final_df = pd.DataFrame(['If you can't see the images please make sure you are signed in to your Twinkl account on Google & you have access to the Shared Drives you are searching :)