1. Import data:
import numpy as np import pandas as pd from category_encoders import TargetEncoder, MEstimateEncoder import warnings warnings. filter warnings('ignore') data_df = pd.read_csv()#add path data_df. head()
Since the data has no categorical variables, a new categorical variable SalePrice1 is added for testing and running through the data
data_df["SalePrice1"]=pd.DataFrame(np.random.randint(0,2,size=(data_df.shape[0], 1)),columns=['test']) data_df
2. MeanEncoder class definition code
import numpy as np import pandas as pd from sklearn.model_selection import StratifiedKFold from sklearn.model_selection import KFold from itertools import product class MeanEncoder: def __init__(self, categorical_features, n_splits=5, target_type='classification', prior_weight_func=None): """ :param categorical_features: list of str, the name of the categorical columns to encode :param n_splits: the number of splits used in mean encoding :param target_type: str, 'regression' or 'classification' :param prior_weight_func: a function that takes in the number of observations, and outputs prior weight When a dict is passed, the default exponential decay function will be used: k: the number of observations needed for the posterior to be weighted equally as the prior f: larger f --> smaller slope """ self.categorical_features = categorical_features self.n_splits = n_splits self. learned_stats = {} if target_type == 'classification': self. target_type = target_type self. target_values = [] else: self. target_type = 'regression' self. target_values = None if isinstance(prior_weight_func, dict): self.prior_weight_func = eval('lambda x: 1 / (1 + np.exp((x - k) / f))', dict(prior_weight_func, np=np)) elif callable(prior_weight_func): self.prior_weight_func = prior_weight_func else: self.prior_weight_func = lambda x: 1 / (1 + np.exp((x - 2) / 1)) @staticmethod def mean_encode_subroutine(X_train, y_train, X_test, variable, target, prior_weight_func): X_train = X_train[[variable]].copy() X_test = X_test[[variable]].copy() if target is not None: nf_name = '{}_pred_{}'. format(variable, target) X_train['pred_temp'] = (y_train == target).astype(int) # classification else: nf_name = '{}_pred'. format(variable) #print(y_train) #LotShape_pred X_train['pred_temp'] = y_train # regression prior = X_train['pred_temp']. mean() #print(prior) ## col_avg_y = X_train.groupby(by=variable, axis=0)['pred_temp'].agg([("mean",'mean'),('beta', 'size')]) col_avg_y['beta'] = prior_weight_func(col_avg_y['beta']) print(col_avg_y['beta']) col_avg_y[nf_name] = col_avg_y['beta'] * prior + (1 - col_avg_y['beta']) * col_avg_y['mean'] col_avg_y.drop(['beta', 'mean'], axis=1, inplace=True) # print(col_avg_y) nf_train = X_train.join(col_avg_y, on=variable)[nf_name].values nf_test = X_test.join(col_avg_y, on=variable).fillna(prior, inplace=False)[nf_name].values return nf_train, nf_test, prior, col_avg_y def fit_transform(self, X, y): """ :param X: pandas DataFrame, n_samples * n_features :param y: pandas Series or numpy array, n_samples :return X_new: the transformed pandas DataFrame containing mean-encoded categorical features """ X_new = X.copy() print(X_new) if self. target_type == 'classification': skf = StratifiedKFold(self.n_splits) else: skf = KFold(self.n_splits) if self. target_type == 'classification': self. target_values = sorted(set(y)) print(self. target_values) self.learned_stats = {'{}_pred_{}'.format(variable, target): [] for variable, target in product(self. categorical_features, self. target_values)} print(self. learned_stats) for variable, target in product(self. categorical_features, self. target_values): print(variable, target) nf_name = '{}_pred_{}'. format(variable, target) X_new.loc[:, nf_name] = np.nan for large_ind, small_ind in skf.split(y, y): nf_large, nf_small, prior, col_avg_y = MeanEncoder.mean_encode_subroutine( X_new.iloc[large_ind], y.iloc[large_ind], X_new.iloc[small_ind], variable, target, self.prior_weight_func) X_new.iloc[small_ind, -1] = nf_small self.learned_stats[nf_name].append((prior, col_avg_y)) print(X_new) else: self.learned_stats = {'{}_pred'.format(variable): [] for variable in self.categorical_features} for variable in self.categorical_features: print(variable) nf_name = '{}_pred'. format(variable) X_new.loc[:, nf_name] = np.nan print(skf. split(y, y)) for large_ind, small_ind in skf.split(y, y): print(large_ind. shape, small_ind. shape) nf_large, nf_small, prior, col_avg_y = MeanEncoder.mean_encode_subroutine( X_new.iloc[large_ind], y.iloc[large_ind], X_new.iloc[small_ind], variable, None, self.prior_weight_func) X_new.iloc[small_ind, -1] = nf_small self.learned_stats[nf_name].append((prior, col_avg_y)) return X_new def transform(self, X): """ :param X: pandas DataFrame, n_samples * n_features :return X_new: the transformed pandas DataFrame containing mean-encoded categorical features """ X_new = X.copy() if self. target_type == 'classification': for variable, target in product(self. categorical_features, self. target_values): nf_name = '{}_pred_{}'. format(variable, target) X_new[nf_name] = 0 for prior, col_avg_y in self.learned_stats[nf_name]: X_new[nf_name] + = X_new[[variable]].join(col_avg_y, on=variable).fillna(prior, inplace=False)[ nf_name] X_new[nf_name] /= self.n_splits else: for variable in self.categorical_features: nf_name = '{}_pred'. format(variable) X_new[nf_name] = 0 for prior, col_avg_y in self.learned_stats[nf_name]: X_new[nf_name] + = X_new[[variable]].join(col_avg_y, on=variable).fillna(prior, inplace=False)[ nf_name] X_new[nf_name] /= self.n_splits return X_new
3. Call the function and run through the fit_transform() method
data_df[“SalePrice1”] is composed of 0 and 1, which can be considered as a classification, so in the definition code of the MeanEncoder class above, target_type=’classification’
enc = MeanEncoder(['LotShape', 'SaleType']) enc=enc.fit_transform(data_df[['LotShape', 'SaleType']], data_df["SalePrice1"]) enc
The result is as follows:
data_df[“SalePrice”] consists of numerical values, which can be considered as regression, so in the definition code of the MeanEncoder class above, let target_type=’regression’
enc = MeanEncoder(['LotShape', 'SaleType']) enc=enc.fit_transform(data_df[['LotShape', 'SaleType']], data_df["SalePrice"]) enc
The result is as follows: