Tensorflow 卷积神经网络、模型维度reshape、模型测试

from __future__ import print_function
import tensorflow.keras as keras
from tensorflow.keras.layers import Input, Dense,  Flatten, add
from tensorflow.keras.layers import Conv2D, Activation, AveragePooling2D, MaxPooling2D
from tensorflow.keras import backend as K
import tensorflow as tf
from tensorflow.keras.models import Model

import dataUtils, plotUtils
import tensorflow_addons as tfa

config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.compat.v1.Session(config=config)

def res_block(x, channels, i):
    if i == 1:
        strides = (1, 1)
        x_add = x
    else:
        strides = (2, 2)
        x_add = Conv2D(channels, kernel_size=(3, 3), activation='relu', padding='same', strides=strides)(x)

    x = Conv2D(channels, kernel_size=(3, 3), activation='relu', padding='same')(x)
    x = Conv2D(channels, kernel_size=(3, 3), padding='same', strides=strides)(x)
    x = add([x, x_add])
    Activation(K.relu)(x)
    return x


def build_model(input_shape, num_classes):

    inpt = Input(shape=input_shape)
    x = keras.layers.Lambda(lambda y: tf.reshape(y, [-1, 32, 32, 1]))(inpt)
    x = keras.layers.Lambda(lambda y: tfa.image.gaussian_filter2d(y, [9, 9], 1, padding='SYMMETRIC'))(x)
    x = keras.layers.Lambda(lambda y: tfa.image.sharpness(y, 0.15))(x)
    x = keras.layers.Lambda(lambda y: tf.image.per_image_standardization(y))(x)
    x = keras.layers.GaussianNoise(0.2)(x)

    x = Conv2D(16, kernel_size=(7, 7), activation='relu', input_shape=input_shape, padding='same')(x)
    # x = MaxPooling2D(pool_size=(3, 3), strides=(2, 2))(x)

    for i in range(2):
        x = res_block(x, 16, i)

    for i in range(2):
        x = res_block(x, 32, i)

    x = AveragePooling2D(pool_size=(7, 7))(x)
    x = Flatten()(x)
    for n in range(3):
        x = Dense(32, activation='relu')(x)
        x = keras.layers.Dropout(0.5)(x)

    x = Dense(num_classes, activation='softmax')(x)

    model = Model(inputs=inpt, outputs=x)
    OP = keras.optimizers.Adam(lr=0.00015, epsilon=None, decay=0.0)
    model.compile(loss=keras.losses.sparse_categorical_crossentropy,
                  optimizer=OP,
                  metrics=['acc'])
    return model

def main(train_path, test_path, outpath, epochs, batch_size, num_classes):
    epochs = int(epochs)
    batch_size = int(batch_size)
    label = "position"
    train_x, train_y, test_x, test_y = dataUtils.get_data(train_path, label, train_ratio=0.9, data_augmentation='False')
    print(train_x, len(test_x))
    for index in range(len(train_x)):
        print(train_x[index])
        print(train_y[index])
    # test_x, test_y, _, _ = dataUtils.get_train_and_test_data(test_path, label, train_ratio=0.95)

    model = build_model([1024, 1], num_classes)
    dynamic_LR = keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1,
                                                   patience=10, verbose=0, mode='auto',
                                                   epsilon=0.0001, cooldown=0, min_lr=0)
    early_stopping = keras.callbacks.EarlyStopping(monitor='val_loss', patience=7, mode='auto')
    checkpoint_cb = keras.callbacks.ModelCheckpoint(outpath+"/model.h5", save_best_only=True)

    history = model.fit(train_x, train_y,
                        batch_size=batch_size,
                        epochs=epochs,
                        verbose=1,
                        callbacks=[dynamic_LR, early_stopping, checkpoint_cb],
                        validation_split=0.3)

    # train_x2, train_y2, test_x2, test_y2 = dataUtils.get_data(test_path, label, train_ratio=0.99)
    # y_pre = model.predict(train_x2[1:50, :])
    # print(y_pre)
    # print(train_y2[1:50])
    loss1, acc1 = model.evaluate(train_x, train_y)
    print('Res test_2,acc:{:5.2f}%'.format(100*acc1))

    plotUtils.plot_history(history, outpath)
    tf.saved_model.save(model, outpath+'/tensorflow/ncz/1')
    plotUtils.plot_confusion_matrix(model, test_x, test_y, outpath)


if __name__ == '__main__':
    train_path = "Data/all_data.csv"
    test_path = "Data/sdata.csv"
    outpath = 'Data'
    batch_size = 50
    num_classes = 3
    epochs = 12
    main(train_path, test_path, outpath, epochs, batch_size, num_classes)

import itertools
import matplotlib.pyplot as plt
import numpy as np
from sklearn import metrics


def plot_history(history, path):
    acc = history.history['acc']
    val_acc = history.history['val_acc']
    loss = history.history['loss']
    val_loss = history.history['val_loss']
    epochs = range(1, len(acc) + 1)
    plt.plot(epochs, acc, 'bo', label='Training acc')
    plt.plot(epochs, val_acc, 'b', label='Validation acc')
    plt.title('Training and validation accuracy')
    plt.savefig(path+"/Training_and_validation_accuracy.png")
    plt.legend()
    plt.figure()
    plt.plot(epochs, loss, 'bo', label='Training loss')
    plt.plot(epochs, val_loss, 'b', label='Validation loss')
    plt.title('Training and validation loss')
    plt.savefig(path + "/Training_and_validation_loss.png")
    plt.legend()
    plt.show()

def plotmatrix(path, cm, classes,
                          title='Confusion matrix',
                          cmap=plt.cm.jet):
    cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, '{:.2f}'.format(cm[i, j]), horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")
    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')
    plt.savefig(path + "/Confusion_matrix.png")
    plt.show()


def plot_confusion_matrix(model, x_test, y_test, path):
    predictions = model.predict(x_test)
    y_pred = np.argmax(predictions, axis=1)
    matrix = metrics.confusion_matrix(y_test, y_pred)
    plt.figure()
    plotmatrix(path, matrix, range(np.max(y_test)+1))

import pandas as pd
import numpy as np
import re
import tensorflow as tf
from scipy.ndimage import gaussian_filter1d



def get_raw_data_from_csv(filepath, label, normalization=False, data_augmentation=False):

    dataframes = pd.read_csv(filepath)
    x = []
    y = []

    for i in dataframes.index:
        data = dataframes.loc[i]
        item_1 = list(map(float, re.findall(r"\d+\.?\d*", data['data'])))
        item = np.asarray(item_1)
        x.append(item.astype(np.float32))
        if data[label] == 1:
            y.append(0)
        elif data[label] == 2:
            y.append(1)
        elif data[label] == 3:
            y.append(2)
        # elif data[label] == 4 or data[label] == 0:
        #     y.append(4)

        # if data_augmentation:
        #     new_item = np.fliplr(np.reshape(item, (32, 32)))
        #     new_item = np.flipud(new_item)
        #     new_item = new_item.reshape(1024)
        #     x.append(new_item.astype(np.float32))
        #     if data[label] == 1:
        #         y.append(1)
        #     elif data[label] == 2:
        #         y.append(2)
        #     elif data[label] == 3:
        #         y.append(3)
        #     elif data[label] == 4 or data[label] == 0:
        #         y.append(4)
        #
        #     new_item_2 = np.fliplr(np.reshape(item, (32, 32)))
        #     new_item_2 = new_item_2.reshape(1024)
        #     x.append(new_item_2.astype(np.float32))
        #     if data[label] == 1:
        #         y.append(1)
        #     elif data[label] == 2:
        #         y.append(2)
        #     elif data[label] == 3:
        #         y.append(3)
        #     elif data[label] == 4 or data[label] == 0:
        #         y.append(4)
    x = np.array(x).astype(np.float32)
    y = np.array(y)
    return x, y


def get_data(filepath, label, normalization=False, shuffle=True, train_ratio=0.8, data_augmentation='False'):

    x, y = get_raw_data_from_csv(filepath, label, normalization, data_augmentation)

    if shuffle:
        permutation = np.random.permutation(y.shape[0])
        x = x[permutation]
        y = y[permutation]

    train_total = x.shape[0]
    train_nums = int(train_total * train_ratio)

    x_train = x[0:train_nums]
    y_train = y[0:train_nums]

    x_test = x[train_nums:train_total]
    y_test = y[train_nums:train_total]

    return x_train, y_train, x_test, y_test


if __name__ == '__main__':
    data_path = "Data/nczData202241.csv"
    get_data(data_path, 'position', train_ratio=0.8, data_augmentation='True')