import gradio as gr import json from flask import jsonify from sentence_transformers import SentenceTransformer, InputExample, util from codeScripts.utils import save_json, load_json, create_file_path, remove from plentas import Plentas import pandas as pd import zipfile import os import shutil from datetime import datetime import tablib from pathlib import Path def Main(uploadedFile, txtFileInput, orthographyPercentage, syntaxPercentage, semanticPercentage, studentsRange): error = "" excelPath = None copySpanishDictionaries() try: if not txtFileInput: error="Por favor seleccione un archivo con las preguntas y respuestas" return [error, excelPath] else: txtFileInput = txtFileInput.name configuration = readQATextFile(txtFileInput) configuration["ortographyPercentage"] = float(orthographyPercentage) configuration["syntaxPercentage"] = float(syntaxPercentage) configuration["semanticPercentage"] = float(semanticPercentage) if studentsRange == "": studentsRange = "All" configuration["students"] = studentsRange if not uploadedFile: error="Por favor seleccione el .zip con las respuestas de los alumnos" return [error, excelPath] else: uploadedFilePath = uploadedFile.name config_json = load_json("configV2.json") answersDict = None try: answersDict = answersTodict(uploadedFilePath) except Exception as ex: error = "Error in answersTodict: " + str(ex) return [error, excelPath] teacherJson = None try: teacherJson = createTeacherJson(configuration) except Exception as ex: error = "Error in createTeacherJson: " + str(ex) return [error, excelPath] try: # #configuring plentas methodology response = Plentas(config_json[0], [answersDict, teacherJson]) except Exception as ex: error = "Error configuring: " + str(ex) return [error, excelPath] try: # # #overwriting the custom settings for the settings from the api response.setApiSettings(configuration) except Exception as ex: error = "Error setting: " + str(ex) return [error, excelPath] try: #print("Processing!") modelResult = response.processApiData() except Exception as ex: error = "Error processing: " + str(ex) return [error, excelPath] # modelJson = json.dumps(modelResult) #print(modelResult) outputFilePath='./output/' + str(datetime.now().microsecond) + '_plentas_output.xlsx' excelPath = exportResultToExcelFile(modelResult, outputFilePath) except Exception as ex: error = "Error exporting to Excel: " + str(ex) return [error, excelPath] def exportResultToExcelFile(modelResult, outputFilePath): try: excelData = [] studentsArray = modelResult[0] index = 0 for item in studentsArray: #print("ITEM - " + str(item)) studentData = item[index] excelData.append(studentData) index+= 1 #tableResults = tablib.Dataset(headers=('ID', 'SimilitudSpacy', 'SimilitudBert', 'NotaSemanticaSpacy', 'NotaSemanticaBert', 'NotaSintaxis', 'NotaOrtografia','NotaTotalSpacy','NotaTotalBert','Feedback')) #tableResults = tablib.Dataset(headers=('ID', 'SumaTotalSpacy', 'SumaTotaldBert', 'NotaSemanticaSpacy', 'NotaSemanticaBert', 'NotaSintaxis', 'NotaOrtografia','NotaTotalSpacy','NotaTotalBert','Feedback')) tableResults = tablib.Dataset(headers=('ID', 'RespuestaAlumno', 'NotaSemanticaSpacy', 'NotaSemanticaBert', 'NotaSintaxis', 'NotaOrtografia','NotaTotalSpacy','NotaTotalBert','Feedback')) tableResults.json=json.dumps(excelData) tableExport=tableResults.export('xlsx') #outputFilePath = './output/' + str(datetime.now().microsecond) + '_plentas_output.xlsx' #outputFilePath = './output/plentas_output.xlsx' with open(outputFilePath, 'wb') as f: # open the xlsx file f.write(tableExport) # write the dataset to the xlsx file f.close() except Exception as ex: print("Error exportando Excel: "+ str(ex)) return outputFilePath def copySpanishDictionaries(): try: shutil.copy("./assets/hunspell_dictionaries/es_ES/es_ES.aff", "/home/user/.local/lib/python3.9/site-packages/hunspell/dictionaries/es_ES.aff") shutil.copy("./assets/hunspell_dictionaries/es_ES/es_ES.dic", "/home/user/.local/lib/python3.9/site-packages/hunspell/dictionaries/es_ES.dic") except Exception as ex: print("Error copying dictionaries: " + str(ex)) def readQATextFile(qaTextFilePath): configuration = {} f = open(qaTextFilePath, 'r') lines = f.readlines() count = 0 qCount=1 q = "" a = "" while count < len(lines): if q == "" or q == "\n": q = lines[count] count += 1 continue if a == "" or a == "\n": a = lines[count] count += 1 if q != "" and a != "": configuration["minip" + str(qCount)] = q configuration["minir" + str(qCount)] = a qCount += 1 q = "" a = "" return configuration def createTeacherJson(configuration): """ This function extracts the information about the subquestions and subanswers and puts them in the correct format. Inputs: config: The configured info from the api. Outputs: teachersJson: The generated dictionary with the subquestions. """ teachersJson = {"enunciado": "", "minipreguntas":[], "keywords":""} #5 is the maximum number of permitted subquestions in the configuration2 page for i in range(5): try: teachersJson["minipreguntas"].append({ "minipregunta": configuration["minip" + str(i+1)], "minirespuesta": configuration["minir" + str(i+1)] }) except: break return teachersJson def extractZipData(ruta_zip): """ This function extracts the students's answers from the zip file (the one the teacher has in the task section). Inputs: ruta_zip: The path inherited from answersTodict """ #defining the path where the extracted info is to be stored ruta_extraccion = create_file_path("StudentAnswers/", doctype= 1) #extracting the info archivo_zip = zipfile.ZipFile(ruta_zip, "r") try: archivo_zip.extractall(pwd=None, path=ruta_extraccion) #archivo_zip.extract(pwd=None, path=ruta_extraccion) except: pass archivo_zip.close() def removeHtmlFromString(string): """ This function removes the html tags from the student's response. Inputs: -string: The student's response Outputs: -new_string: The filtered response """ string = string.encode('utf-8', 'replace') string = string.decode('utf-8', 'replace') new_string = "" skipChar = 0 for char in string: if char == "<": skipChar = 1 elif char == ">": skipChar = 0 else: if not skipChar: new_string = new_string+char new_string = new_string.encode('utf-8', 'replace') new_string = new_string.decode('utf-8', 'replace') return new_string def answersTodict(zip_path): """ This function extracts the students's answers and stacks them in one specific format so that it can be processed next. Inputs: ruta_zip: The path where the zip file is stored Outputs: studentAnswersDict: The dictionary with all the responses """ # path remove('api/StudentAnswers/') #extracting the data extractZipData(zip_path) studentAnswersDict = [] indx2=0 #stacking the information of each extracted folder for work_folder in os.listdir(create_file_path("StudentAnswers/", doctype= 1)): #print("work_folder: " + work_folder) for student, indx in zip(os.listdir(create_file_path("StudentAnswers/" + work_folder, doctype= 1)), range(len(os.listdir(create_file_path("StudentAnswers/" + work_folder, doctype= 1))))): student_name = student.split("(") student_name = student_name[0] #print("student: " + str(student) + " - index: " + str(indx)) try: #opening the file #fichero1 = create_file_path("StudentAnswers/" + work_folder + "/" + student+ "/" + 'Adjuntos del envio/', doctype= 1) fichero1 = create_file_path("StudentAnswers/" + work_folder + "/" + student + "/" + student+'_submissionText.html', doctype= 1) #where the actual response is if os.path.exists(fichero1): #print("Fichero: "+str(fichero1)) #fichero = open(create_file_path("StudentAnswers/" + work_folder + "/" + student + "/" + 'Adjuntos del envio/Respuesta enviada', doctype= 1), encoding='utf-8') #fichero = create_file_path("StudentAnswers/" + work_folder + "/" + student + "/" + 'Adjuntos del envio/Respuesta enviada', doctype= 1) #if os.path.exists(fichero): # fichero = open(create_file_path("StudentAnswers/" + work_folder + "/" + student + "/" + 'Adjuntos del envio/Respuesta enviada', doctype= 1), encoding='utf-8') #else: fichero = open(create_file_path("StudentAnswers/" + work_folder + "/" + student + "/" + student+'_submissionText.html', doctype= 1), encoding='utf-8') #print("fichero abierto") #reading it lineas = fichero.readlines() textoFichero = "" if (len(lineas) > 0): #textoFichero = lineas[0] string_without_line_breaks = "" for line in lineas: stripped_line = line.rstrip() string_without_line_breaks += stripped_line textoFichero = string_without_line_breaks #print("texto: " + textoFichero) #removing html textoFichero = removeHtmlFromString(str(textoFichero.encode("utf-8"))) #saving it studentAnswersDict.append({"respuesta":textoFichero, "hashed_id":student_name, "TableIndex":indx}) #print("fichero procesado") elif os.path.exists(create_file_path("StudentAnswers/" + work_folder, doctype= 1)) : #print("Entra por acá2") student_name2 = work_folder.split("_") student_name = student_name2[0] student_id2=student_name2[1] student_assingsubmission = student_name2[2] student_response = student_name2[3] #print("Sigue acá2") #print("StudentResponse"+str(student_response)) if student_response=='onlinetext': # print("Fichero: " + "StudentAnswers/" + work_folder+"/onlinetext.html") fichero = open(create_file_path("StudentAnswers/" + work_folder+"/onlinetext.html", doctype= 1), encoding='utf-8') lineas = fichero.readlines() #removing html #lineas[0] = removeHtmlFromString(lineas[0]) textoFichero="" #removing html if (len(lineas) > 0): string_without_line_breaks = "" for line in lineas: stripped_line = line.rstrip() string_without_line_breaks += stripped_line textoFichero = string_without_line_breaks #textoFichero = lineas[0] #lineas[0] = removeHtmlFromString(lineas[0]) textoFichero = removeHtmlFromString(str(textoFichero.encode("utf-8"))) #print(textoFichero) #saving it indx2+=1 studentAnswersDict.append({"respuesta":textoFichero, "hashed_id":student_name, "TableIndex":indx2}) #break #elif student_response!='file' or None: # print("Fichero no encontrado") # studentAnswersDict.append({"respuesta":'', "hashed_id":student_name, "TableIndex":indx2}) except: #studentAnswersDict.append({"respuesta":"", "hashed_id":student_name, "TableIndex":indx}) print("Error buscando ficheros:"+fichero) #print("DICT" + json.dumps(studentAnswersDict)) #saving the final dictionary save_json(create_file_path('ApiStudentsDict.json', doctype= 1),studentAnswersDict) return studentAnswersDict zipFileInput = gr.inputs.File(label="1. Selecciona el .ZIP con las respuestas de los alumnos") txtFileInput = gr.inputs.File(label="2. Selecciona el .txt con las preguntas y respuestas correctas. Escriba una pregunta en una sola línea y debajo la respuesta en la línea siguiente.") orthographyPercentage = gr.inputs.Textbox(label="Ortografía",lines=1, placeholder="0",default=0.1, numeric=1) syntaxPercentage = gr.inputs.Textbox(label="Sintaxis",lines=1, placeholder="0",default=0.1,numeric=1) semanticPercentage = gr.inputs.Textbox(label="Semántica",lines=1, placeholder="0",default=0.8, numeric=1) studentsRange = gr.inputs.Textbox(label="Estudiantes a evaluar",lines=1, placeholder="Dejar vacío para evaluar todos") #dataFrameOutput = gr.outputs.Dataframe(headers=["Resultados"], max_rows=20, max_cols=None, overflow_row_behaviour="paginate", type="pandas", label="Resultado") labelOutput = gr.outputs.Label(num_top_classes=None, type="auto", label="Output") labelError = gr.outputs.Label(num_top_classes=None, type="auto", label="Errores") downloadExcelButton = gr.outputs.File('Resultados') iface = gr.Interface(fn=Main , inputs=[zipFileInput, txtFileInput, orthographyPercentage, syntaxPercentage, semanticPercentage, studentsRange] , outputs=[labelError, downloadExcelButton] , title = "PLENTAS" ) #iface.launch(share = False,enable_queue=True, show_error =True, server_port= 7861) iface.launch(share = False,enable_queue=True, show_error =True)