|
import pandas as pd
|
|
import numpy as np
|
|
import matplotlib.pyplot as plt
|
|
import seaborn as sns
|
|
from scipy.stats.mstats import winsorize
|
|
from sklearn.preprocessing import StandardScaler,MinMaxScaler
|
|
from sklearn.metrics import root_mean_squared_error
|
|
from scipy.stats import yeojohnson
|
|
import evaluationer
|
|
from sklearn.model_selection import train_test_split as tts
|
|
def detect_outliers(df,num_cols):
|
|
global outlier_df,zscore_cols,outlier_indexes,iqr_cols
|
|
outlier_df = pd.DataFrame({"method" :[],"columns name":[],"upper limit":[],
|
|
"lower limit":[],"no of Rows":[],"percentage outlier":[]})
|
|
if type(num_cols) == list:
|
|
if len(num_cols)!=0:
|
|
num_cols = num_cols
|
|
else:
|
|
num_cols = df.select_dtypes(exclude = "object").columns.tolist()
|
|
else:
|
|
if num_cols.tolist() != None:
|
|
num_cols = num_cols
|
|
else:
|
|
num_cols = df.select_dtypes(exclude = "object").columns.tolist()
|
|
zscore_cols = []
|
|
iqr_cols = []
|
|
outlier_indexes =[]
|
|
for col in num_cols:
|
|
skewness = df[col].skew()
|
|
if -0.5 <= skewness <= 0.5:
|
|
method = "zscore"
|
|
zscore_cols.append(col)
|
|
|
|
else:
|
|
method = "iqr"
|
|
iqr_cols.append(col)
|
|
if len(zscore_cols) >0:
|
|
for col in zscore_cols:
|
|
mean = df[col].mean()
|
|
std = df[col].std()
|
|
ul = mean + (3*std)
|
|
ll = mean - (3*std)
|
|
mask = (df[col] < ll) | (df[col] > ul)
|
|
temp = df[mask]
|
|
|
|
Zscore_index = temp.index.tolist()
|
|
outlier_indexes.extend(Zscore_index)
|
|
|
|
if len(temp)>0:
|
|
|
|
temp_df = pd.DataFrame({"method" : ["ZScore"],
|
|
"columns name" : [col],
|
|
"upper limit" : [round(ul,2)],
|
|
"lower limit" :[ round(ll,2)],
|
|
"no of Rows" : [len(temp)],
|
|
"percentage outlier" : [round(len(temp)*100/len(df),2)]})
|
|
|
|
outlier_df = pd.concat([outlier_df,temp_df]).reset_index(drop = True)
|
|
|
|
else:
|
|
print("No columns for Zscore method")
|
|
|
|
|
|
if len(iqr_cols) >0:
|
|
for col in iqr_cols:
|
|
q3 = df[col].quantile(.75)
|
|
q1 = df[col].quantile(.25)
|
|
IQR = q3 -q1
|
|
ul = q3 + 1.5*IQR
|
|
ll = q1 - 1.5*IQR
|
|
mask = (df[col] < ll) | (df[col] > ul)
|
|
temp = df[mask]
|
|
|
|
IQR_index = temp.index.tolist()
|
|
outlier_indexes.extend(IQR_index)
|
|
|
|
if len(temp)>0:
|
|
list(outlier_indexes).append(list(IQR_index))
|
|
|
|
temp_df1 = pd.DataFrame({"method" : ["IQR"],
|
|
"columns name" : [col],
|
|
"upper limit" : [round(ul,2)],
|
|
"lower limit" : [round(ll,2)],
|
|
"no of Rows": [len(temp)],
|
|
"percentage outlier" : [round((len(temp)*100/len(df)),2)]
|
|
})
|
|
|
|
outlier_df = pd.concat([outlier_df,temp_df1]).reset_index(drop = True)
|
|
|
|
else:
|
|
print("No columns for IQR method")
|
|
|
|
|
|
outlier_indexes = list(set(outlier_indexes))
|
|
|
|
return outlier_df,outlier_indexes
|
|
|
|
|
|
def outlier_handling(df,y,model,outlier_indexes = [],outlier_cols = None ,method = root_mean_squared_error,test_size = 0.2, random_state = 42,eva = "reg"):
|
|
num_col = df.select_dtypes(exclude = "O").columns
|
|
|
|
global outliers_dropped_df,log_transformed_df,sqrt_transformed_df,yeo_johnson_transformed_df,rank_transformed_df
|
|
global std_scaler_df,winsorize_transformed_df,inverse_log_transformed_winsorize_df,inverse_sqrt_transformed_winsorize_df,minmaxscaler_df
|
|
if eva == "reg":
|
|
if len(outlier_indexes) ==0:
|
|
print("no outlier indexes passed")
|
|
outliers_dropped_df = df.copy()
|
|
else:
|
|
outliers_dropped_df = df.drop(index =outlier_indexes)
|
|
|
|
if outlier_cols != None:
|
|
|
|
if df[outlier_cols][df[outlier_cols] <0].sum().sum() == 0:
|
|
log_transformed_df = df.copy()
|
|
log_transformed_df[outlier_cols] = np.log(log_transformed_df[outlier_cols] + 1e-5)
|
|
sqrt_transformed_df = df.copy()
|
|
sqrt_transformed_df[outlier_cols] = np.sqrt(sqrt_transformed_df[outlier_cols] + 1e-5)
|
|
inverse_log_transformed_winsorize_df = log_transformed_df.copy()
|
|
inverse_sqrt_transformed_winsorize_df = sqrt_transformed_df.copy()
|
|
for column in outlier_cols:
|
|
inverse_log_transformed_winsorize_df[column] = np.exp(winsorize(inverse_log_transformed_winsorize_df[column], limits=[0.05, 0.05]))
|
|
inverse_sqrt_transformed_winsorize_df[column] = (winsorize(inverse_sqrt_transformed_winsorize_df[column], limits=[0.05, 0.05]))**2
|
|
else:
|
|
print("df have values less than zero")
|
|
std_scaler_df = df.copy()
|
|
std_scaler_df[outlier_cols] = StandardScaler().fit_transform(std_scaler_df[outlier_cols])
|
|
|
|
minmaxscaler_df = df.copy()
|
|
minmaxscaler_df[outlier_cols] = MinMaxScaler().fit_transform(minmaxscaler_df[outlier_cols])
|
|
|
|
yeo_johnson_transformed_df = df.copy()
|
|
for column in outlier_cols:
|
|
try:
|
|
yeo_johnson_transformed_df[column], lambda_ = yeojohnson(yeo_johnson_transformed_df[column])
|
|
|
|
except :
|
|
yeo_johnson_transformed_df[column] = yeo_johnson_transformed_df[column]
|
|
|
|
print(f"Yeo-Johnson transformation failed for column '{column}'. Original data used.")
|
|
|
|
rank_transformed_df = df.copy()
|
|
rank_transformed_df[outlier_cols] = rank_transformed_df[outlier_cols].rank()
|
|
winsorize_transformed_df = df.copy()
|
|
for column in outlier_cols:
|
|
winsorize_transformed_df[column] = winsorize(winsorize_transformed_df[column], limits=[0.05, 0.05])
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
if df[num_col][df[num_col] <0].sum().sum() == 0:
|
|
log_transformed_df = df.copy()
|
|
log_transformed_df[num_col] = np.log(log_transformed_df[num_col] + 1e-5)
|
|
sqrt_transformed_df = df.copy()
|
|
sqrt_transformed_df[num_col] = np.sqrt(sqrt_transformed_df[num_col] + 1e-5)
|
|
inverse_log_transformed_winsorize_df = log_transformed_df.copy()
|
|
inverse_sqrt_transformed_winsorize_df = sqrt_transformed_df.copy()
|
|
for column in num_col:
|
|
inverse_log_transformed_winsorize_df[column] = np.exp(winsorize(inverse_log_transformed_winsorize_df[column], limits=[0.05, 0.05]))
|
|
inverse_sqrt_transformed_winsorize_df[column] = (winsorize(inverse_sqrt_transformed_winsorize_df[column], limits=[0.05, 0.05]))**2
|
|
else:
|
|
|
|
print("df have values less than zero")
|
|
|
|
std_scaler_df = df.copy()
|
|
std_scaler_df[outlier_cols] = StandardScaler().fit_transform(std_scaler_df[outlier_cols])
|
|
|
|
minmaxscaler_df = df.copy()
|
|
minmaxscaler_df[outlier_cols] = MinMaxScaler().fit_transform(minmaxscaler_df[outlier_cols])
|
|
|
|
yeo_johnson_transformed_df = df.copy()
|
|
for column in num_col:
|
|
try:
|
|
yeo_johnson_transformed_df[column], lambda_ = yeojohnson(yeo_johnson_transformed_df[column])
|
|
|
|
except :
|
|
yeo_johnson_transformed_df[column] = yeo_johnson_transformed_df[column]
|
|
|
|
print(f"Yeo-Johnson transformation failed for column '{column}'. Original data used.")
|
|
|
|
rank_transformed_df = df.copy()
|
|
rank_transformed_df[num_col] = rank_transformed_df[num_col].rank()
|
|
winsorize_transformed_df = df.copy()
|
|
for column in num_col:
|
|
winsorize_transformed_df[column] = winsorize(winsorize_transformed_df[column], limits=[0.05, 0.05])
|
|
|
|
if (df[num_col][df[num_col] <0].sum().sum() == 0):
|
|
outlier_handled_df = [std_scaler_df,minmaxscaler_df,outliers_dropped_df,log_transformed_df,sqrt_transformed_df,yeo_johnson_transformed_df,
|
|
rank_transformed_df,winsorize_transformed_df,inverse_log_transformed_winsorize_df,inverse_sqrt_transformed_winsorize_df]
|
|
|
|
outlier_handled_df_name = ["std_scaler_df","minmaxscaler_df","outliers_dropped_df", "log_transformed_df","sqrt_transformed_df", "yeo_johnson_transformed_df","rank_transformed_df","winsorize_transformed_df",
|
|
"inverse_log_transformed_winsorize_df", "inverse_sqrt_transformed_winsorize_df"]
|
|
elif df[outlier_cols][df[outlier_cols] <0].sum().sum() == 0:
|
|
outlier_handled_df = [std_scaler_df,minmaxscaler_df,outliers_dropped_df,log_transformed_df,sqrt_transformed_df,yeo_johnson_transformed_df,
|
|
rank_transformed_df,winsorize_transformed_df,inverse_log_transformed_winsorize_df,inverse_sqrt_transformed_winsorize_df]
|
|
|
|
outlier_handled_df_name = ["std_scaler_df","minmaxscaler_df","outliers_dropped_df","log_transformed_df", "sqrt_transformed_df","yeo_johnson_transformed_df","rank_transformed_df",
|
|
"winsorize_transformed_df","inverse_log_transformed_winsorize_df","inverse_sqrt_transformed_winsorize_df"]
|
|
|
|
else:
|
|
outlier_handled_df = [std_scaler_df,minmaxscaler_df,outliers_dropped_df,yeo_johnson_transformed_df,rank_transformed_df,winsorize_transformed_df]
|
|
|
|
outlier_handled_df_name = ["std_scaler_df","minmaxscaler_df","outliers_dropped_df","yeo_johnson_transformed_df","rank_transformed_df","winsorize_transformed_df"]
|
|
|
|
for j,i in enumerate(outlier_handled_df):
|
|
X_train, X_test, y_train, y_test = tts(i,y[i.index],test_size = test_size, random_state = random_state)
|
|
evaluationer.evaluation(f"{outlier_handled_df_name[j]}",X_train,X_test,y_train,y_test,model,root_mean_squared_error,eva)
|
|
|
|
|
|
return evaluationer.reg_evaluation_df , outlier_handled_df,outlier_handled_df_name
|
|
elif eva =="class":
|
|
|
|
std_scaler_df = df.copy()
|
|
|
|
std_scaler_df.loc[:,:] = StandardScaler().fit_transform(std_scaler_df.loc[:,:])
|
|
|
|
minmaxscaler_df = df.copy()
|
|
minmaxscaler_df.loc[:,:] = MinMaxScaler().fit_transform(minmaxscaler_df.loc[:,:])
|
|
|
|
rank_transformed_df = df.copy()
|
|
rank_transformed_df = rank_transformed_df.rank()
|
|
|
|
outlier_handled_df = [std_scaler_df,minmaxscaler_df,rank_transformed_df]
|
|
outlier_handled_df_name = ["std_scaler_df","minmaxscaler_df","rank_transformed_df"]
|
|
|
|
for j,i in enumerate(outlier_handled_df):
|
|
|
|
X_train, X_test, y_train, y_test = tts(i,y[i.index],test_size = test_size, random_state = random_state)
|
|
evaluationer.evaluation(f"{outlier_handled_df_name[j]}", X_train,X_test,y_train,y_test,model,root_mean_squared_error,eva = "class")
|
|
return evaluationer.classification_evaluation_df, outlier_handled_df,outlier_handled_df_name
|
|
|
|
|
|
|
