用keras使用glove预训练的词向量来构建实验的embedding矩阵-以Jigsaw Unintended Bias in Toxicity Classification比赛baseline为例

数据加载

import numpy as np  # linear algebra
import pandas as pd  # data processing, CSV file I/O (e.g. pd.read_csv)
import os
# print(os.listdir("../input"))
# print(os.listdir("../input/crawl300d2m"))
# Any results you write to the current directory are saved as output.
import numpy as np
import pandas as pd
import os
import gc
import logging
import datetime
import warnings
import pickle
from keras.callbacks import EarlyStopping, ModelCheckpoint, LearningRateScheduler
from keras.layers import Input, Dense, Embedding, SpatialDropout1D, Dropout, add, concatenate
from keras.layers import CuDNNLSTM, Bidirectional, GlobalMaxPooling1D, GlobalAveragePooling1D
from keras.preprocessing import text, sequence
from keras.losses import binary_crossentropy
from keras import backend as K
import keras.layers as L
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints, optimizers, layers

from keras.models import Model
from keras.optimizers import Adam
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import KFold
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences

COMMENT_TEXT_COL = 'comment_text'
EMB_MAX_FEAT = 300
MAX_LEN = 220
MAX_FEATURES = 100000
BATCH_SIZE = 512
NUM_EPOCHS = 4
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 512
NUM_MODELS = 2
EMB_PATHS = [
    #'data/crawl-300d-2M.vec',
    'data/glove.840B.300d.txt'
]
JIGSAW_PATH = 'data/'


def get_logger():
    FORMAT = '[%(levelname)s]%(asctime)s:%(name)s:%(message)s'
    logging.basicConfig(format=FORMAT)
    logger = logging.getLogger('main')
    logger.setLevel(logging.DEBUG)
    return logger
logger = get_logger()
############################################################################################

def custom_loss(y_true, y_pred):#计算loss
    return binary_crossentropy(K.reshape(y_true[:, 0], (-1, 1)), y_pred) * y_true[:, 1]


def load_data():
    logger.info('Load train and test data')
    train = pd.read_csv(os.path.join(JIGSAW_PATH, 'train.csv'), index_col='id')
    test = pd.read_csv(os.path.join(JIGSAW_PATH, 'test.csv'), index_col='id')
    return train, test

数据预处理

def perform_preprocessing(train, test):
    logger.info('data preprocessing')
    punct_mapping = {
   
   "_": " "