Stacking ile Kelimelerden Dil Tahmini

import pandas as pd
import numpy as np
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.metrics import f1_score, accuracy_score, confusion_matrix, precision_score, recall_score
from sklearn.linear_model import RidgeClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.model_selection import KFold, StratifiedKFold, train_test_split
from lightgbm import LGBMClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
import lightgbm as lgb
from sklearn.svm import LinearSVC
import gc

seed = 1001

df = pd.read_csv("language_detection.csv")

df.head()

df["is_turkish"].value_counts()

df["is_turkish"].value_counts(normalize = True)

df["Words"].apply(lambda x: len(x)).value_counts()

df['Vowels'] = df.Words.str.lower().str.count(r'[aeiou]')
df['Consonant'] = df.Words.str.lower().str.count(r'[a-z]') - df['Vowels']
df["Len"] = df.Words.apply(lambda x: len(x))

df.head()

word = False
word_tfidf = False

count_para_word = {
    "analyzer": "word",
    "dtype": np.float32,
}

count_para_char = {
    "analyzer": "char_wb",
    "dtype": np.float32, # RAM için bir diğer optimize edilebilecek argüman.
    "ngram_range": (1, 6) # maksimum harf sayımız 6 olduğu için üst sınır 6
}

if word:
    vectorizer = CountVectorizer(
        max_features=50000, 
        **count_para_word)
else:
    vectorizer = CountVectorizer(
        max_features=50000, 
        **count_para_char)

tfidf_para_word = {
    "analyzer": "word",
    "sublinear_tf": True,
    "dtype": np.float32,
    "norm": 'l2',
    "min_df": 5,
    "max_df": 0.9,
    "smooth_idf": False
}

tfidf_para_char = {
    "analyzer": "char_wb",
    "dtype": np.float32,
    "ngram_range": (1, 6) # maksimum harf sayımız 6 olduğu için üst sınır 6
}

if word_tfidf:
    vectorizer_tfidf = TfidfVectorizer(
                max_features=5000,
                **tfidf_para_word)
else:
    vectorizer_tfidf = TfidfVectorizer(
                max_features=50000,
                **tfidf_para_char)

vectorizer.fit(df["Words"])

vectorizer_tfidf.fit(df["Words"])

train_vector, test_vector = train_test_split(df, test_size = 0.2, stratify = df["is_turkish"], random_state = seed)

train_vector = train_vector[(train_vector.is_turkish == 1) | (
    train_vector.is_turkish == 0).sample(frac=0.4, random_state=seed)].sample(
        frac=1, random_state=seed).reset_index(drop=True)

y_train = train_vector["is_turkish"].values
y_test = test_vector["is_turkish"].values

train_vector_ = vectorizer.transform(train_vector["Words"])
test_vector_ = vectorizer.transform(test_vector["Words"])

train_vector_tfidf = vectorizer_tfidf.transform(train_vector["Words"])
test_vector_tfidf = vectorizer_tfidf.transform(test_vector["Words"])

from scipy.sparse import hstack, csr_matrix

train_vector = hstack([
    train_vector_,
    train_vector_tfidf,
    csr_matrix(train_vector.Vowels.values.reshape(-1, 1)),
    csr_matrix(train_vector.Consonant.values.reshape(-1, 1)),
    csr_matrix(train_vector.Len.values.reshape(-1, 1))
])

test_vector = hstack([
    test_vector_,
    test_vector_tfidf,
    csr_matrix(test_vector.Vowels.values.reshape(-1, 1)),
    csr_matrix(test_vector.Consonant.values.reshape(-1, 1)),
    csr_matrix(test_vector.Len.values.reshape(-1, 1))
])

train_vector = train_vector.tocsr()
test_vector = test_vector.tocsr()

train_vector.shape, y_train.shape

 test_vector.shape, y_test.shape

del train_vector_, test_vector_
del train_vector_tfidf, test_vector_tfidf

class SklearnWrapper(object):
    def __init__(self, clf, seed=0, params=None, seed_bool = True):
        if(seed_bool == True):
            params['random_state'] = seed
        self.clf = clf(**params)

    def train(self, x_train, y_train):
        self.clf.fit(x_train, y_train)

    def predict(self, x):
        return self.clf.predict(x)
        
    def predict_proba(self, x):
        return self.clf.predict_proba(x)
    
    
def get_oof(clf, x_train, y, x_test, prob = False):
    oof_train = np.zeros((ntrain,))
    oof_test = np.zeros((ntest,))
    oof_test_skf = np.empty((NFOLDS, ntest))

    for i, (train_index, test_index) in enumerate(kf.split(x_train, y)):
        x_tr = x_train[train_index]
        y_tr = y[train_index]
        x_te = x_train[test_index]
        y_te = y[test_index]

        clf.train(x_tr, y_tr)
        if prob:
            oof_train[test_index] = clf.predict_proba(x_te)[:,1]
            oof_test_skf[i, :] = clf.predict_proba(x_test)[:,1]
        else:
            oof_train[test_index] = clf.predict(x_te)
            oof_test_skf[i, :] = clf.predict(x_test)
        print("Fold:", i+1)
        print("F1", np.round(f1_score(y_te, np.round(oof_train[test_index])),6))
        print("\n")
        
        
    oof_test[:] = oof_test_skf.mean(axis=0)
    return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1)

def scorer(y_true, y_pred, is_return = False):
    if is_return:
        return [f1_score(y_true, np.round(y_pred)), accuracy_score(y_true, np.round(y_pred)), recall_score(y_true, np.round(y_pred)), precision_score(y_true, np.round(y_pred))]
    else:
        print("F1: {:.4f}".format(f1_score(y_true, np.round(y_pred))))
        print("Accuracy: {:.4f}".format(accuracy_score(y_true, np.round(y_pred))))
        print("Recall: {:.4f}".format(recall_score(y_true, np.round(y_pred))))
        print("Precision: {:.4f}".format(precision_score(y_true, np.round(y_pred))))
        print("AUC: {:.4f}".format(roc_auc_score(y_true, y_pred)))
        print((confusion_matrix(y_true, np.round(y_pred))))

NFOLDS = 5

ntrain = train_vector.shape[0]
ntest = test_vector.shape[0]
kf = StratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed)

ridge_params = {'alpha':250.0, 'fit_intercept':True, 'normalize':True, 'copy_X':True,
                'max_iter':None, 'tol':0.001, 'solver':'auto', 'random_state':seed,
               "class_weight": "balanced"
               }

ridge = SklearnWrapper(clf=RidgeClassifier, seed = seed, params = ridge_params)

%%time
ridge_oof_train, ridge_oof_test = get_oof(ridge, train_vector, y_train, test_vector, prob=False)

scorer(ridge_oof_train, y_train)

scorer(ridge_oof_test, y_test)

svm_params = {"C": 3, "max_iter": 10000, "class_weight": "balanced"}

svm = SklearnWrapper(clf=LinearSVC, seed = seed, params = svm_params)

%%time
svm_oof_train, svm_oof_test = get_oof(svm, train_vector.toarray(), y_train, test_vector.toarray(), prob=False)

scorer(svm_oof_train, y_train)

scorer(svm_oof_test, y_test)

lgbm_params =  {
    'task': 'train',
    'boosting_type': 'gbdt',
    'objective': 'binary',
    'metric': 'binary_error',
    'num_leaves': 17,
    "max_depth": -1,
    'feature_fraction': 0.95,
    'bagging_fraction': 0.9,
    'bagging_freq': 8,
    'learning_rate': 0.075,
    'verbose': 0,
    "n_estimator": 50,
    "pos_bagging_fraction": 0.8,
    "neg_bagging_fraction": 0.6,
}

%%time
lgbc = SklearnWrapper(clf=LGBMClassifier, seed = seed, params = lgbm_params)
lgbc_oof_train, lgbc_oof_test = get_oof(lgbc, train_vector, y_train, test_vector, prob=True)

scorer(lgbc_oof_train, y_train)

scorer(lgbc_oof_test, y_test)

rf_params = {"n_jobs":12,
            "n_estimators": 380,
            "class_weight": "balanced",
            "min_samples_split": 5}

%%time
rf = SklearnWrapper(clf=RandomForestClassifier, seed = seed, params = rf_params)
rf_oof_train, rf_oof_test = get_oof(rf, train_vector, y_train, test_vector, prob=True)

scorer(rf_oof_train, y_train)

scorer(rf_oof_test, y_test)

rf_2_params = {"n_jobs":12,
               "max_depth": 15,
               "n_estimators": 500,
               "criterion": "gini",
               "min_samples_split": 5,
               "min_samples_leaf": 3,
               "class_weight": "balanced"}

%%time
rf_2 = SklearnWrapper(clf=RandomForestClassifier, seed = seed, params = rf_2_params)
rf_2_oof_train, rf_2_oof_test = get_oof(rf_2, train_vector, y_train, test_vector, prob=True)

scorer(rf_2_oof_train, y_train)

scorer(rf_2_oof_test, y_test)

lr_params = {"n_jobs":12,
            "solver": "saga",
             "C": 10,
            "max_iter": 1000}

%%time
lr = SklearnWrapper(clf=LogisticRegression, seed = seed, params = lr_params)
lr_oof_train, lr_oof_test = get_oof(lr, train_vector, y_train, test_vector, prob = True)

scorer(lr_oof_train, y_train)

scorer(lr_oof_test, y_test)

lr_2_params = {
            "solver": "liblinear",
    "class_weight": "balanced",
            "max_iter": 750,
            "C":25,
              "penalty": "l1"}

%%time
lr_2 = SklearnWrapper(clf=LogisticRegression, seed = seed, params = lr_2_params)
lr_2_oof_train, lr_2_oof_test = get_oof(lr_2, train_vector, y_train, test_vector, prob = True)

scorer(lr_2_oof_train, y_train)

scorer(lr_2_oof_test, y_test)

def threshold_search(y_true, y_proba):
    best_threshold = 0
    best_score = 0
    for threshold in [i * 0.01 for i in range(100)]:
        score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
        if score > best_score:
            best_threshold = threshold
            best_score = score
    search_result = {'threshold': best_threshold, 'f1': best_score}
    return search_result

ridge_thr = threshold_search(y_train, ridge_oof_train)
ridge_thr

lgb_thr = threshold_search(y_train, lgbc_oof_train)
lgb_thr

rf_thr = threshold_search(y_train, rf_oof_train)
rf_thr

rf2_thr = threshold_search(y_train, rf_2_oof_train)
rf2_thr

lr_thr = threshold_search(y_train, lr_oof_train)
lr_thr

lr2_thr = threshold_search(y_train, lr_2_oof_train)
lr2_thr

svm_thr = threshold_search(y_train, svm_oof_train)
svm_thr

train_df = np.concatenate((
    ridge_oof_train,
    (lr_oof_train > lr_thr["threshold"]).astype(int),
    (lr_2_oof_train > lr2_thr["threshold"]).astype(int),
    (rf_oof_train > rf_thr["threshold"]).astype(int),
    (rf_2_oof_train > rf2_thr["threshold"]).astype(int),
    (lgbc_oof_train > lgb_thr["threshold"]).astype(int),
    svm_oof_train
),
                          axis=1)

test_df = np.concatenate((
    ridge_oof_test,
    (lr_oof_test > lr_thr["threshold"]).astype(int),
    (lr_2_oof_test > lr2_thr["threshold"]).astype(int),
    (rf_oof_test > rf_thr["threshold"]).astype(int),
    (rf_2_oof_test > rf2_thr["threshold"]).astype(int),
    (lgbc_oof_test > lgb_thr["threshold"]).astype(int),
    svm_oof_test
),
                         axis=1)

# Yeni veri setimiz
pd.DataFrame(train_df).head()

y_train.shape, train_df.shape

dtrain = lgb.Dataset(train_df, label=y_train)

def lgb_f1_score(y_hat, data):
    y_true = data.get_label()
    y_hat = np.round(y_hat)
    return 'f1', f1_score(y_true, y_hat), True

lgbm_params =  {
    'task': 'train',
    'boosting_type': 'gbdt',
    'objective': 'binary',
    'metric': 'auc',
    'num_leaves': 11,
    "max_depth": 4,
    'feature_fraction': 0.9,
    'bagging_fraction': 0.95,
    'bagging_freq': 12,
    'learning_rate': 0.05,
    "scale_pos_weight":  0.9,
    'verbose': 0,
    "pos_bagging_fraction": 0.8,
    "neg_bagging_fraction": 0.6,
}

cv = lgb.cv(lgbm_params,
            dtrain,
            num_boost_round=1000,
            folds=kf,
            verbose_eval=5,
            early_stopping_rounds=20,
            feval=lgb_f1_score,
            eval_train_metric=False)

cv["f1-mean"][-1]

best_n = len(cv["f1-mean"])
best_n

model = lgb.train(lgbm_params, dtrain, num_boost_round=int(best_n * 1.1))

test_preds = model.predict(test_df)

scorer(y_test, np.round(test_preds))

results_df = pd.DataFrame([scorer(ridge_oof_test, y_test, is_return=True),
scorer(svm_oof_test, y_test, is_return=True),
scorer(lgbc_oof_test, y_test, is_return=True),
scorer(rf_oof_test, y_test, is_return=True),
scorer(rf_2_oof_test, y_test, is_return=True),
scorer(lr_oof_test, y_test, is_return=True),
scorer(lr_2_oof_test, y_test, is_return=True),
scorer(y_test, np.round(test_preds), is_return=True)], columns=["F1", "Accuracy", "Recall", "Precision"], index = ["Ridge", "SVM", "LGB", "RF", "RF2", "LR", "LR2", "Stacked Model"])
results_df