# -*- coding: utf-8 -*- """old_models.ipynb Automatically generated by Colaboratory. Original file is located at https://colab.research.google.com/drive/1Oc7A5TaGLg1qkYXzf0qLGIe0_ZxyAnXE This notebook contains Feature selection with Chi-Square test, Logistic Regression with TFIDF as well as Bidirectional LSTM with gensim to classifies a given tweet into depressive or non-depressive ones. """ from google.colab import drive drive.mount('/content/drive') ## Import required libraries ## warnings import warnings warnings.filterwarnings("ignore") ## for data import numpy as np import pandas as pd ## for plotting import matplotlib.pyplot as plt import seaborn as sns ## TF-IDF from sklearn.feature_extraction.text import TfidfVectorizer ## T-Sne from yellowbrick.text import TSNEVisualizer from sklearn import manifold ## Train-Test Split from sklearn.model_selection import train_test_split ## Feature selection from sklearn import feature_selection ## libraraies for classification from sklearn.pipeline import Pipeline import sklearn.metrics as skm from sklearn.metrics import confusion_matrix, accuracy_score from sklearn.linear_model import LogisticRegression from sklearn.neighbors import KNeighborsClassifier from sklearn.svm import SVC from sklearn.tree import DecisionTreeClassifier from sklearn.neural_network import MLPClassifier from sklearn.ensemble import RandomForestClassifier ## for saving model import pickle ## detokenization from nltk.tokenize.treebank import TreebankWordDetokenizer ## for word embedding with gensim import gensim import gensim.downloader as gensim_api from gensim.models import Word2Vec from gensim.models import KeyedVectors from keras.preprocessing.text import Tokenizer from keras.preprocessing.sequence import pad_sequences ## for word embedding with Spacy # import spacy # import en_core_web_lg ## for deep learning from keras.models import load_model from keras.models import Model, Sequential from keras.callbacks import EarlyStopping, ModelCheckpoint from keras.layers import Conv1D, Dense, Input, LSTM, Embedding, Dropout, Activation, MaxPooling1D from tensorflow.keras import models, layers, preprocessing as kprocessing from tensorflow.keras import backend as K import tensorflow as tf import keras from keras.layers import Lambda import tensorflow as tf from keras.models import model_from_json ## for bert language model #import transformers """## Loading the dataset:""" df_all = pd.read_csv("/content/drive/MyDrive/NLP/Depression_Detection/data_cleaning/processed_data/processed_data.csv", sep='\t', encoding='utf-8') df_all """## Feature selection In order to drop some columns and reduce the matrix dimensionality, we can carry out some Feature Selection, the process of selecting a subset of relevant variables. I will proceed as follows: 1. treat each category as binary (for example, the “depressive” category is 1 for the depressive tweets and 0 for non_depressive); 2. perform a Chi-Square test to determine whether a feature and the (binary) target are independent; 3. keep only the features with a certain p-value from the Chi-Square test. This snippet of code is derived from https://towardsdatascience.com/text-classification-with-nlp-tf-idf-vs-word2vec-vs-bert-41ff868d1794 """ y = y_train X_names = cv.get_feature_names() p_value_limit = 0.95 df_features = pd.DataFrame() for cat in np.unique(y): chi2, p = feature_selection.chi2(X_train_tfidf, y==cat) df_features = df_features.append(pd.DataFrame( {"feature":X_names, "score":1-p, "y":cat})) df_features = df_features.sort_values(["y","score"], ascending=[True,False]) df_features = df_features[df_features["score"]>p_value_limit] X_names = df_features["feature"].unique().tolist() print(len(X_names)) """I reduced the number of features from 20018 to 688 by keeping the most statistically relevant ones. Let’s print some:""" for cat in np.unique(y): print("# {}:".format(cat)) print(" . selected features:", len(df_features[df_features["y"]==cat])) print(" . top features:", ",".join(df_features[df_features["y"]==cat]["feature"].values[:10])) print(" ") """## Logistic Regression with TFIDF: ### Spliting data to train and test datasets: """ ## split dataset to train and test X_train, X_test, y_train, y_test = train_test_split(df_all['clean_text'], df_all['label'], test_size=0.3, random_state= 42) X_train.shape, X_test.shape, y_train.shape, y_test.shape """### TF-IDF TF-IDF (term frequency and inverse document frequency): """ ## Creating the TF-IDF model cv = TfidfVectorizer() cv.fit(X_train.to_list()) dic_vocabulary = cv.vocabulary_ X_train_tfidf = cv.transform(X_train.to_list()) X_test_tfidf = cv.transform(X_test.to_list()) cv.inverse_transform(X_test_tfidf[0]) X_train_tfidf.shape # ## Adding clean tweets to a list called corpus # corpus = [] # corpus = [x for x in df_train['clean_text']] # # corpus = df_train["clean_text"] """The feature matrix X_train_tfidf has a shape of 16,464 (Number of documents in training) x 20018 (Length of vocabulary) and it’s pretty sparse:""" sns.heatmap(X_train_tfidf.todense()[:,np.random.randint(0,X_train_tfidf.shape[1],100)]==0, vmin=0, vmax=1, cbar=False).set_title('Sparse Matrix Sample') """In order to know the position of a certain word, we can look it up in the vocabulary:""" word = "mental" dic_vocabulary[word] """Build a scikit-learn pipeline: a sequential application of a list of transformations and a final estimator. Putting the Tf-Idf vectorizer and Logistic Regression classifier in a pipeline allows us to transform and predict test data in just one step.""" # classifier = LogisticRegression(solver='liblinear', penalty='l1') # ## pipeline # model = Pipeline([("vectorizer", cv), # ("classifier", classifier)]) # ## train classifier # model["classifier"].fit(X_train, y_train) # ## test # predicted = model.predict(X_test) # predicted_prob = model.predict_proba(X_test) # ## creating the instance of the models lr = LogisticRegression(solver='liblinear', penalty='l1') ## fitting the model print(lr.fit(X_train_tfidf, y_train.to_list())) ## Save the Modle to file in the current working directory LogisticReg = "/content/drive/MyDrive/NLP/Depression_Detection/modeling/model_LogReg.pkl" with open(LogisticReg, 'wb') as file: pickle.dump(lr, file) ## Load the Model back from file with open(LogisticReg, 'rb') as file: lr = pickle.load(file) lr ## Test y_pred_lr = lr.predict(X_test_tfidf) probs = lr.predict_proba(X_test_tfidf) classes = np.unique(y_test.to_list()) y_test_array = pd.get_dummies(y_test, drop_first=False).values """## Evaluate the performance: * **Accuracy:** the fraction of predictions the model got right. * **Confusion Matrix:** a summary table that breaks down the number of correct and incorrect predictions by each class. * **ROC:** a plot that illustrates the true positive rate against the false positive rate at various threshold settings. The area under the curve (AUC) indicates the probability that the classifier will rank a randomly chosen positive observation higher than a randomly chosen negative one. * **Precision:** the fraction of relevant instances among the retrieved instances. * **Recall:** the fraction of the total amount of relevant instances that were actually retrieved. """ def conf_matrix_acc(y_true, y_pred): ## Plot confusion matrix cm = confusion_matrix(y_true, y_pred) fig, ax = plt.subplots() sns.heatmap(cm, annot=True, fmt='d', ax=ax, cmap=plt.cm.Blues, cbar=False) ax.set(xlabel="Pred", ylabel="True", xticklabels=classes, yticklabels=classes, title="Confusion matrix") plt.yticks(rotation=0) print("=========================================") print(f'Accuracy score is : {accuracy_score(y_true, y_pred)}') print("=========================================") print("Detail:") print(skm.classification_report(y_true, y_pred)) ## Plot ROC and precision-recall curve def roc_precision_auc(): fig, ax = plt.subplots(nrows=1, ncols=2) ## Plot roc for i in range(len(classes)): fpr, tpr, thresholds = skm.roc_curve(y_test_array[:,i], probs[:,i]) ax[0].plot(fpr, tpr, lw=3, label='{0} (area={1:0.2f})'.format(classes[i], skm.auc(fpr, tpr)) ) ax[0].plot([0,1], [0,1], color='navy', lw=3, linestyle='--') ax[0].set(xlim=[-0.05,1.0], ylim=[0.0,1.05], xlabel='False Positive Rate', ylabel="True Positive Rate (Recall)", title="Receiver operating characteristic") ax[0].legend(loc="lower right") ax[0].grid(True) ## Plot precision-recall curve for i in range(len(classes)): precision, recall, thresholds = skm.precision_recall_curve( y_test_array[:,i], probs[:,i]) ax[1].plot(recall, precision, lw=3, label='{0} (area={1:0.2f})'.format(classes[i], skm.auc(recall, precision)) ) ax[1].set(xlim=[0.0,1.05], ylim=[0.0,1.05], xlabel='Recall', ylabel="Precision", title="Precision-Recall curve") ax[1].legend(loc="best") ax[1].grid(True) plt.show() #plt.savefig('/content/drive/MyDrive/NLP/Depression_Detection/modeling/ROC_Precision_LR.png') plt.savefig('/content/drive/MyDrive/NLP/Depression_Detection/modeling/ROC_Precision_SVM.png') ## AUC score print(f'AUC score is : {skm.roc_auc_score(Y_test, probs[:,1])}') conf_matrix_acc(y_test.to_list(),y_pred_lr) roc_precision_auc() """## Bidirectional LSTM: In Python, you can load a pre-trained Word Embedding model from genism-data like this: """ nlp_pre = gensim_api.load("word2vec-google-news-300") word = "anxiety" fig = plt.figure() ## word embedding tot_words = [word] + [tupla[0] for tupla in nlp_pre.most_similar(word, topn=20)] X = nlp_pre[tot_words] ## pca to reduce dimensionality from 300 to 3 pca = manifold.TSNE(perplexity=40, n_components=3, init='pca') X = pca.fit_transform(X) ## create dtf dtf_ = pd.DataFrame(X, index=tot_words, columns=["x","y","z"]) dtf_["input"] = 0 dtf_["input"].iloc[0:1] = 1 ## plot 3d from mpl_toolkits.mplot3d import Axes3D ax = fig.add_subplot(111, projection='3d') ax.scatter(dtf_[dtf_["input"]==0]['x'], dtf_[dtf_["input"]==0]['y'], dtf_[dtf_["input"]==0]['z'], c="black") ax.scatter(dtf_[dtf_["input"]==1]['x'], dtf_[dtf_["input"]==1]['y'], dtf_[dtf_["input"]==1]['z'], c="red") ax.set(xlabel=None, ylabel=None, zlabel=None, xticklabels=[], yticklabels=[], zticklabels=[]) for label, row in dtf_[["x","y","z"]].iterrows(): x, y, z = row ax.text(x, y, z, s=label) """Instead of using a pre-trained model, I am going to fit my own Word2Vec on the training data corpus with gensim. Before fitting the model, the corpus needs to be transformed into a list of lists of n-grams. In this particular case, I’ll try to capture unigrams (“york”), bigrams (“new york”), and trigrams (“new york city”).""" ## split dataset dtf_train, dtf_test = train_test_split(df_all, test_size=0.3) ## get target y_train = dtf_train["label"].values y_test = dtf_test["label"].values corpus = [] corpus = [x for x in dtf_train['clean_text']] ## create list of lists of unigrams lst_corpus = [] for string in corpus: lst_words = str(string).split() lst_grams = [" ".join(lst_words[i:i+1]) for i in range(0, len(lst_words), 1)] lst_corpus.append(lst_grams) ## detect bigrams and trigrams bigrams_detector = gensim.models.phrases.Phrases(lst_corpus, delimiter=" ".encode(), min_count=5, threshold=10) bigrams_detector = gensim.models.phrases.Phraser(bigrams_detector) trigrams_detector = gensim.models.phrases.Phrases(bigrams_detector[lst_corpus], delimiter=" ".encode(), min_count=5, threshold=10) trigrams_detector = gensim.models.phrases.Phraser(trigrams_detector) """When fitting the Word2Vec, you need to specify: * the target size of the word vectors, I’ll use 300; * the window, or the maximum distance between the current and predicted word within a sentence, I’ll use the mean length of text in the corpus; * the training algorithm, I’ll use skip-grams (sg=1) as in general it has better results. """ ## fit w2v nlp = gensim.models.word2vec.Word2Vec(lst_corpus, size=300, window=8, min_count=1, sg=1, iter=30) """We have our embedding model, so we can select any word from the corpus and transform it into a vector.""" word = "anxiety" nlp[word].shape """We can even use it to visualize a word and its context into a smaller dimensional space (2D or 3D) by applying any dimensionality reduction algorithm (i.e. TSNE).""" word = "anxiety" fig = plt.figure() ## word embedding tot_words = [word] + [tupla[0] for tupla in nlp.most_similar(word, topn=20)] X = nlp[tot_words] ## pca to reduce dimensionality from 300 to 3 pca = manifold.TSNE(perplexity=40, n_components=3, init='pca') X = pca.fit_transform(X) ## create dtf dtf_ = pd.DataFrame(X, index=tot_words, columns=["x","y","z"]) dtf_["input"] = 0 dtf_["input"].iloc[0:1] = 1 ## plot 3d from mpl_toolkits.mplot3d import Axes3D ax = fig.add_subplot(111, projection='3d') ax.scatter(dtf_[dtf_["input"]==0]['x'], dtf_[dtf_["input"]==0]['y'], dtf_[dtf_["input"]==0]['z'], c="black") ax.scatter(dtf_[dtf_["input"]==1]['x'], dtf_[dtf_["input"]==1]['y'], dtf_[dtf_["input"]==1]['z'], c="red") ax.set(xlabel=None, ylabel=None, zlabel=None, xticklabels=[], yticklabels=[], zticklabels=[]) for label, row in dtf_[["x","y","z"]].iterrows(): x, y, z = row ax.text(x, y, z, s=label) """The word vectors can be used in a neural network as weights in the follwing procedure: 1. Transform the corpus into padded sequences of word ids to get a feature matrix. 2. Create an embedding matrix so that the vector of the word with id N is located at the Nth row. 3. Build a neural network with an embedding layer that weighs every word in the sequences with the corresponding vector. **Feature Engineering:** by transforming the same preprocessed corpus (list of lists of n-grams) given to the Word2Vec into a list of sequences using tensorflow/keras: """ ## tokenize text tokenizer = kprocessing.text.Tokenizer(lower=True, split=' ', oov_token="NaN", filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n') tokenizer.fit_on_texts(lst_corpus) dic_vocabulary = tokenizer.word_index ## create sequence lst_text2seq= tokenizer.texts_to_sequences(lst_corpus) ## padding sequence X_train = kprocessing.sequence.pad_sequences(lst_text2seq, maxlen=35, padding="post", truncating="post") X_train.shape """The feature matrix X_train has a shape of 16559 x 35 (Number of sequences x Sequences max length). Let’s visualize it:""" sns.heatmap(X_train==0, vmin=0, vmax=1, cbar=False) plt.show() """Every text in the corpus is now an id sequence with length 35. For instance, if a text had 20 tokens in it, then the sequence is composed of 20 ids + 15 0s, which is the padding element (while the id for word not in the vocabulary is 1) Let’s print how a text from the train set has been transformed into a sequence with the padding and the vocabulary. """ i = 8 ## list of text: ["I like this", ...] len_txt = len(dtf_train["clean_text"].iloc[i].split()) print("from: ", dtf_train["clean_text"].iloc[i], "| len:", len_txt) ## sequence of token ids: [[1, 2, 3], ...] len_tokens = len(X_train[i]) print("to: ", X_train[i], "| len:", len(X_train[i])) ## vocabulary: {"I":1, "like":2, "this":3, ...} print("check: ", dtf_train["clean_text"].iloc[i].split()[0], " -- idx in vocabulary -->", dic_vocabulary[dtf_train["clean_text"].iloc[i].split()[0]]) print("vocabulary: ", dict(list(dic_vocabulary.items())[0:5]), "... (padding element, 0)") corpus = dtf_test["clean_text"] ## create list of n-grams lst_corpus = [] for string in corpus: lst_words = str(string).split() lst_grams = [" ".join(lst_words[i:i+1]) for i in range(0, len(lst_words), 1)] lst_corpus.append(lst_grams) ## detect common bigrams and trigrams using the fitted detectors lst_corpus = list(bigrams_detector[lst_corpus]) lst_corpus = list(trigrams_detector[lst_corpus]) ## text to sequence with the fitted tokenizer lst_text2seq = tokenizer.texts_to_sequences(lst_corpus) ## padding sequence X_test = kprocessing.sequence.pad_sequences(lst_text2seq, maxlen=35, padding="post", truncating="post") X_test.shape sns.heatmap(X_test==0, vmin=0, vmax=1, cbar=False) plt.show() """We’ve got our X_train and X_test, now we need to create the embedding matrix that will be used as a weight matrix in the neural network.""" ## start the matrix (length of vocabulary x vector size) with all 0s embeddings = np.zeros((len(dic_vocabulary)+1, 300)) for word,idx in dic_vocabulary.items(): ## update the row with vector try: embeddings[idx] = nlp[word] ## if word not in model then skip and the row stays all 0s except: pass embeddings.shape """That code generates a matrix of shape 20,050 x 300 (Length of vocabulary extracted from the corpus x Vector size). It can be navigated by word id, which can be obtained from the vocabulary.""" word = "anxiety" print("dic[word]:", dic_vocabulary[word], "|idx") print("embeddings[idx]:", embeddings[dic_vocabulary[word]].shape, "|vector") """### Deep Learning: It’s finally time to build a deep learning model. I’m going to use the embedding matrix in the first Embedding layer of the neural network that I will build and train to classify the news. Each id in the input sequence will be used as the index to access the embedding matrix. The output of this Embedding layer will be a 2D matrix with a word vector for each word id in the input sequence (Sequence length x Vector size). Let’s use the sentence “I like this article” as an example: My neural network shall be structured as follows: * An Embedding layer that takes the sequences as input and the word vectors as weights, just as described before. * A simple Attention layer that won’t affect the predictions but it’s going to capture the weights of each instance and allow us to build a nice explainer (it isn't necessary for the predictions, just for the explainability, so you can skip it). * Two layers of Bidirectional LSTM to model the order of words in a sequence in both directions. * Two final dense layers that will predict the probability of each category. """ ## code attention layer def attention_layer(inputs, neurons): x = layers.Permute((2,1))(inputs) x = layers.Dense(neurons, activation="softmax")(x) x = layers.Permute((2,1), name="attention")(x) x = layers.multiply([inputs, x]) return x ## input x_in = layers.Input(shape=(35,)) ## embedding x = layers.Embedding(input_dim=embeddings.shape[0], output_dim=embeddings.shape[1], weights=[embeddings], input_length=35, trainable=False)(x_in) ## apply attention x = attention_layer(x, neurons=35) ## 2 layers of bidirectional lstm x = layers.Bidirectional(layers.LSTM(units=35, dropout=0.2, return_sequences=True))(x) x = layers.Bidirectional(layers.LSTM(units=35, dropout=0.2))(x) ## final dense layers x = layers.Dense(64, activation='relu')(x) y_out = layers.Dense(1, activation='sigmoid')(x) ## compile model = models.Model(x_in, y_out) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) model.summary() ## encode y dic_y_mapping = {n:label for n,label in enumerate(np.unique(y_train))} inverse_dic = {v:k for k,v in dic_y_mapping.items()} y_train = np.array([inverse_dic[y] for y in y_train]) ## train training = model.fit(x=X_train, y=y_train, batch_size=256, epochs=30, shuffle=True, verbose=0, validation_split=0.3) ## plot loss and accuracy metrics = [k for k in training.history.keys() if ("loss" not in k) and ("val" not in k)] fig, ax = plt.subplots(nrows=1, ncols=2, sharey=True) ax[0].set(title="Training") ax11 = ax[0].twinx() ax[0].plot(training.history['loss'], color='black') ax[0].set_xlabel('Epochs') ax[0].set_ylabel('Loss', color='black') for metric in metrics: ax11.plot(training.history[metric], label=metric) ax11.set_ylabel("Score", color='steelblue') ax11.legend() ax[1].set(title="Validation") ax22 = ax[1].twinx() ax[1].plot(training.history['val_loss'], color='black') ax[1].set_xlabel('Epochs') ax[1].set_ylabel('Loss', color='black') for metric in metrics: ax22.plot(training.history['val_'+metric], label=metric) ax22.set_ylabel("Score", color="steelblue") plt.savefig('/content/drive/MyDrive/NLP/Depression_Detection/modeling/loss_accuracy_LSTM_3.png') plt.show() # serialize model to JSON model_json = model.to_json() with open("/content/drive/MyDrive/NLP/Depression_Detection/modeling/model.json", "w") as json_file: json_file.write(model_json) # serialize weights to HDF5 model.save_weights("/content/drive/MyDrive/NLP/Depression_Detection/modeling/model.h5") print("Saved model to disk") loaded_model = model_from_json(open("/content/drive/MyDrive/NLP/Depression_Detection/modeling/model.json", "r").read(), custom_objects={'tf': tf}) json_file.close() # load weights into new model loaded_model.load_weights("/content/drive/MyDrive/NLP/Depression_Detection/modeling/model.h5") print("Loaded model from disk") labels_pred = model.predict(X_test) labels_pred = np.round(labels_pred.flatten()) accuracy = accuracy_score(y_test, labels_pred) classes = np.unique(y_test) print("Accuracy: %.2f%%" % (accuracy*100)) def conf_matrix_acc2(y_true, y_pred): ## Plot confusion matrix cm = confusion_matrix(y_test, y_pred) fig, ax = plt.subplots() sns.heatmap(cm, annot=True, fmt='d', ax=ax, cmap=plt.cm.Blues, cbar=False) ax.set(xlabel="Pred", ylabel="True", xticklabels=classes, yticklabels=classes, title="Confusion matrix") plt.yticks(rotation=0) print("=========================================") print(f'Accuracy score is : {accuracy_score(y_true, y_pred)}') print("=========================================") print("Detail:") print(skm.classification_report(y_test, y_pred)) conf_matrix_acc2(y_test, labels_pred) # classes = np.unique(y_test) # y_test_array = pd.get_dummies(y_test, drop_first=False).values # predicted_prob = model.predict_on_batch(X_test) # ## Plot ROC and precision-recall curve # def roc_precision_auc2(): # fig, ax = plt.subplots(nrows=1, ncols=2) # ## Plot roc # for i in range(len(classes)): # fpr, tpr, thresholds = skm.roc_curve(y_test_array[:,i], # predicted_prob[:,i]) # ax[0].plot(fpr, tpr, lw=3, # label='{0} (area={1:0.2f})'.format(classes[i], # skm.auc(fpr, tpr)) # ) # ax[0].plot([0,1], [0,1], color='navy', lw=3, linestyle='--') # ax[0].set(xlim=[-0.05,1.0], ylim=[0.0,1.05], # xlabel='False Positive Rate', # ylabel="True Positive Rate (Recall)", # title="Receiver operating characteristic") # ax[0].legend(loc="lower right") # ax[0].grid(True) # ## Plot precision-recall curve # for i in range(len(classes)): # precision, recall, thresholds = skm.precision_recall_curve( # y_test_array[:,i], probs[:,i]) # ax[1].plot(recall, precision, lw=3, # label='{0} (area={1:0.2f})'.format(classes[i], # skm.auc(recall, precision)) # ) # ax[1].set(xlim=[0.0,1.05], ylim=[0.0,1.05], xlabel='Recall', # ylabel="Precision", title="Precision-Recall curve") # ax[1].legend(loc="best") # ax[1].grid(True) # plt.show() # #plt.savefig('/content/drive/MyDrive/NLP/Depression_Detection/modeling/ROC_Precision_LR.png') # #plt.savefig('/content/drive/MyDrive/NLP/Depression_Detection/modeling/ROC_Precision_LSTM.png') # ## AUC score # print(f'AUC score is : {skm.roc_auc_score(y_test, probs[:,1])}')