File size: 5,077 Bytes
cc0fb09 e5b30ca ab71d04 cd7bf6b cc0fb09 46f16fa cc0fb09 cd7bf6b cc0fb09 5d407bd cc0fb09 5d407bd cc0fb09 6fef1b7 fa5983b 5d407bd cc0fb09 5d407bd cc0fb09 6fef1b7 5d407bd cc0fb09 5d407bd cc0fb09 5d407bd cc0fb09 5d407bd cc0fb09 5d407bd cc0fb09 cd7bf6b cc0fb09 c6ebb8e cd7bf6b cc0fb09 cd7bf6b cc0fb09 5cb79de cc0fb09 5cb79de cc0fb09 9f5b6c4 abe2730 5cb79de 9f5b6c4 5cb79de 9f5b6c4 cc0fb09 f691218 9f5b6c4 69fa6c8 cc0fb09 ed00025 9f5b6c4 cc0fb09 5cb79de 10d367b cc0fb09 3efd370 cd7bf6b 3efd370 cd7bf6b 3efd370 cd7bf6b 3efd370 cd7bf6b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 |
from typing import List, Tuple
from typing_extensions import Literal
import logging
import pandas as pd
from pandas import DataFrame, Series
from utils.config import getconfig
from utils.preprocessing import processingpipeline
import streamlit as st
from setfit import SetFitModel
from transformers import pipeline
## Labels dictionary ###
label_dict = {
'0':'NO',
'1':'YES',
}
def get_target_labels(preds):
"""
Function that takes the numerical predictions as an input and returns a list of the labels.
"""
# Turn into list
preds_list = preds.to_list()
# Get label names
predictions_names=[]
# loop through each prediction
for ele in preds_list:
# see if there is a value 1 and retrieve index
try:
index_of_one = ele.index(1)
except ValueError:
index_of_one = "NA"
# Retrieve the name of the label (if no prediction made = NA)
if index_of_one != "NA":
name = label_dict[index_of_one]
else:
name = "Other"
# Append name to list
predictions_names.append(name)
return predictions_names
@st.cache_resource
def load_targetClassifier(config_file:str = None, classifier_name:str = None):
"""
loads the document classifier using haystack, where the name/path of model
in HF-hub as string is used to fetch the model object.Either configfile or
model should be passed.
1. https://docs.haystack.deepset.ai/reference/document-classifier-api
2. https://docs.haystack.deepset.ai/docs/document_classifier
Params
--------
config_file: config file path from which to read the model name
classifier_name: if modelname is passed, it takes a priority if not \
found then will look for configfile, else raise error.
Return: document classifier model
"""
if not classifier_name:
if not config_file:
logging.warning("Pass either model name or config file")
return
else:
config = getconfig(config_file)
classifier_name = config.get('target','MODEL')
logging.info("Loading classifier")
# Loading classifier
doc_classifier = SetFitModel.from_pretrained("leavoigt/vulnerability_target")
return doc_classifier
@st.cache_data
def target_classification(haystack_doc:pd.DataFrame,
threshold:float = 0.5,
classifier_model:pipeline= None
)->Tuple[DataFrame,Series]:
"""
Text-Classification on the list of texts provided. Classifier provides the
most appropriate label for each text. There labels indicate whether the paragraph
references a specific action, target or measure in the paragraph.
---------
haystack_doc: List of haystack Documents. The output of Preprocessing Pipeline
contains the list of paragraphs in different format,here the list of
Haystack Documents is used.
threshold: threshold value for the model to keep the results from classifier
classifiermodel: you can pass the classifier model directly,which takes priority
however if not then looks for model in streamlit session.
In case of streamlit avoid passing the model directly.
Returns
----------
df: Dataframe with two columns['SDG:int', 'text']
x: Series object with the unique SDG covered in the document uploaded and
the number of times it is covered/discussed/count_of_paragraphs.
"""
logging.info("Working on target/action identification")
haystack_doc['Target Label'] = 'NA'
st.write("haystack_doc")
st.write(haystack_doc)
if not classifier_model:
st.write("No classifier_model")
classifier_model = st.session_state['target_classifier']
st.write("classifier model defined")
# Get predictions
predictions = classifier_model(list(haystack_doc.text))
st.write("predictions made")
st.write(predictions)
# Get labels for predictions
pred_labels = get_target_labels(predictions)
st.write("pred_labels")
st.write(pred_labels)
# Save labels
haystack_doc['Target Label'] = pred_labels
return haystack_doc
# logging.info("Working on action/target extraction")
# if not classifier_model:
# # classifier_model = st.session_state['target_classifier']
# # results = classifier_model(list(haystack_doc.text))
# # labels_= [(l[0]['label'],
# # l[0]['score']) for l in results]
# # df1 = DataFrame(labels_, columns=["Target Label","Target Score"])
# # df = pd.concat([haystack_doc,df1],axis=1)
# # df = df.sort_values(by="Target Score", ascending=False).reset_index(drop=True)
# # df['Target Score'] = df['Target Score'].round(2)
# # df.index += 1
# # # df['Label_def'] = df['Target Label'].apply(lambda i: _lab_dict[i])
|