计算一组Tensor的直方图C算法实现

原创已于 2022-07-25 13:37:27 修改 · 置顶 · 454 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#人工智能

于 2022-07-20 21:24:51 首次发布

人工智能同时被 3 个专栏收录

155 篇文章

订阅专栏

数学

154 篇文章

订阅专栏

算法

101 篇文章

订阅专栏

本文介绍了Tensor量化过程中的KL散度算法，用于衡量信息损失。通过读取和处理150528维度的列向量数据，绘制并分析原始直方图，然后进行数据平移以验证直方图分布的影响。讨论了数据分布对于AI训练和推理的重要性，并展示了不同直方图平移效果。

Tensor量化中，计算信息度损失的的一个重要算法叫做KL散度算法，关于其介绍请参考博客：

模型量化中的KL散度扫盲_papaofdoudou的博客-优快云博客_kl三度

本文介绍其程序实现：

首先构造一组TENSOR向量,维度为150528的列向量。

观察其原始的直方图分布，其分布特点如下图所示：

程序实现：

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stddef.h>
#include <stdint.h>
#include <fcntl.h>
#include <unistd.h>
#include <math.h>
#include <sys/ioctl.h>
 
#define DBG(fmt, ...)   do { printf("%s line %d, "fmt"\n", __func__, __LINE__, ##__VA_ARGS__); } while (0)
#define min(x,y) ({ \
    typeof(x) _x = (x);    \
    typeof(y) _y = (y);    \
    (void) (&_x == &_y);    \
    _x < _y ? _x : _y; })
 
int get_tensor_from_txt_file(char *file_path, float **buf)
{
    int len = 0;
    static float *memory = NULL;
    static int max_len = 10 * 1024 * 1024;
    FILE *fp = NULL;
 
    if(memory == NULL)
    {
        memory = (float*) malloc(max_len * sizeof(float));
    }
    
    if((fp = fopen(file_path, "r")) == NULL)
    {
        DBG("open tensor error.");
        exit(-1);
    }
 
    while(!feof(fp))
    {
        fscanf(fp, "%f", &memory[len ++]);
    }
 
    *buf = (float*)malloc(len * sizeof(float));
    if(len == 0 || *buf == NULL)
    {
        DBG("read tensor error, len %d, *buf %p", len, *buf);
        exit(-1);
    }
    memcpy(*buf, memory, len * sizeof(float));
 
    fclose(fp);
 
    return len;
}
 
int main(int argc, char **argv)
{
    FILE *file;
 
    DBG("in");
 
    if(argc != 3)
    {
        DBG("input error, you should use this program like that: program tensor binsnum.");
        exit(-1);
    }
 
    int tensor0_len;
    float *tensor0_dat;
    int bins = atoi(argv[2]);
    
    tensor0_len = 0;
    tensor0_dat = NULL;
 
    tensor0_len = get_tensor_from_txt_file(argv[1], &tensor0_dat);
    DBG("tensor len %d.", tensor0_len);

    float absmax = 0.f;
    int i = 0;

    for(i = 0; i < tensor0_len + 1; i ++)
    {
        if(fabs(tensor0_dat[i]) > absmax)
            absmax = fabs(tensor0_dat[i]);
    }
    
    DBG("abs = %f.", absmax);

    int *histogram = malloc(bins * sizeof(int));
    float *histogram_norm = malloc(bins * sizeof(float));
    if(histogram == NULL || histogram_norm == NULL)
    {
        DBG("fatal error, malloc histogram failure.");
        exit(-1);
    }

    memset(histogram, 0x00, bins * sizeof(int));

    for(i = 0; i < tensor0_len; i ++)
    {
        if (tensor0_dat[i] == 0.f) continue;

        const int index = min((int)(fabs(tensor0_dat[i]) / absmax * bins), (bins - 1));
        histogram[index] += 1;
    }

    for(i = 0; i < bins; i ++)
    {
        DBG("histogram[%d] = %d.", i, histogram[i]);
    }

    //直方图归一化
    int sum = 0;

    for(i = 0; i < bins; i ++)
    {
        sum += histogram[i];
        histogram_norm[i] = 0.f;
    }

    for(i = 0; i < bins; i ++)
        histogram_norm[i] = (float)((float)histogram[i])/(float)sum;

    for(i = 0; i < bins; i ++)
    {
        DBG("histogram[%d] = %f.", i, histogram_norm[i]);
    }

    DBG("out");
    return 0;
}

运行：

3BINS：

100BINS：

100BINS的数据可视化：

和上面有所差异，原因可能是程序中绝对值的处理，将负值也作为正值处理了。为了验证，我们找到这批数据中的最小值，它是负的，我们将其加上一个偏移，正好变为0，这样所有的值都变为了正数，平移数据范围不会影响直方图的分布，所以我们就可以验证我们的猜测是否正确。

代码：

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stddef.h>
#include <stdint.h>
#include <fcntl.h>
#include <unistd.h>
#include <math.h>
#include <sys/ioctl.h>
 
#define DBG(fmt, ...)   do { printf("%s line %d, "fmt"\n", __func__, __LINE__, ##__VA_ARGS__); } while (0)
#define min(x,y) ({ \
    typeof(x) _x = (x);    \
    typeof(y) _y = (y);    \
    (void) (&_x == &_y);    \
    _x < _y ? _x : _y; })
 
int get_tensor_from_txt_file(char *file_path, float **buf)
{
    int len = 0;
    static float *memory = NULL;
    static int max_len = 10 * 1024 * 1024;
    FILE *fp = NULL;
 
    if(memory == NULL)
    {
        memory = (float*) malloc(max_len * sizeof(float));
    }
    
    if((fp = fopen(file_path, "r")) == NULL)
    {
        DBG("open tensor error.");
        exit(-1);
    }
 
    while(!feof(fp))
    {
        fscanf(fp, "%f", &memory[len ++]);
    }
 
    *buf = (float*)malloc(len * sizeof(float));
    if(len == 0 || *buf == NULL)
    {
        DBG("read tensor error, len %d, *buf %p", len, *buf);
        exit(-1);
    }
    memcpy(*buf, memory, len * sizeof(float));
 
    fclose(fp);
 
    return len;
}
 
int main(int argc, char **argv)
{
    FILE *file;
 
    DBG("in");
 
    if(argc != 3)
    {
        DBG("input error, you should use this program like that: program tensor binsnum.");
        exit(-1);
    }
 
    int tensor0_len;
    float *tensor0_dat;
    int bins = atoi(argv[2]);
    
    tensor0_len = 0;
    tensor0_dat = NULL;
 
    tensor0_len = get_tensor_from_txt_file(argv[1], &tensor0_dat);
    DBG("tensor len %d.", tensor0_len);

    float absmax = 0.f;
    float min = 0.f;
    int i = 0;

    for(i = 0; i < tensor0_len + 1; i ++)
	{
        tensor0_dat[i] += 87.939552;
	}

    for(i = 0; i < tensor0_len + 1; i ++)
    {
        if(fabs(tensor0_dat[i]) > absmax)
            absmax = fabs(tensor0_dat[i]);

        if(tensor0_dat[i] < min)
            min = tensor0_dat[i];
    }
    
    DBG("abs = %f, min %f.", absmax, min);

    int *histogram = malloc(bins * sizeof(int));
    float *histogram_norm = malloc(bins * sizeof(float));
    if(histogram == NULL || histogram_norm == NULL)
    {
        DBG("fatal error, malloc histogram failure.");
        exit(-1);
    }

    memset(histogram, 0x00, bins * sizeof(int));

    for(i = 0; i < tensor0_len; i ++)
    {
        if (tensor0_dat[i] == 0.f) continue;

        const int index = min((int)(fabs(tensor0_dat[i]) / absmax * bins), (bins - 1));
        histogram[index] += 1;
    }

    for(i = 0; i < bins; i ++)
    {
        DBG("histogram[%d] = %d.", i, histogram[i]);
    }

    //直方图归一化
    int sum = 0;

    for(i = 0; i < bins; i ++)
    {
        sum += histogram[i];
        histogram_norm[i] = 0.f;
    }

    for(i = 0; i < bins; i ++)
        histogram_norm[i] = (float)((float)histogram[i])/(float)sum;

    for(i = 0; i < bins; i ++)
    {
        printf("%f\n", histogram_norm[i]);
    }

    DBG("out");
    return 0;
}

这次直方图曲线和本篇开头的直方图符合了：

数据可视化部分的PYTHON代码：

import numpy as np
import linecache
import matplotlib.pyplot as plt

filename = "output.tensor"
cols = 1 # number of column
divided_ch = ' ' # divided_character between numbers

def dat_to_matrix(filename):
    file = open(filename)
    lines = file.readlines()
    rows = len(lines)
    # print(rows)
    # print(lines)
    datamat = np.zeros(rows)
    row = 0

    for line in lines:
        line = line.strip().split(divided_ch) # strip remove block space in line
        datamat[row:] = line[:]
        row += 1

    return datamat


data = dat_to_matrix(filename)
# print(data)
X=np.linspace(0,1,100)      # X轴坐标数据
plt.figure(figsize=(8,6))   # 定义图的大小
plt.plot(X,data)            # 绘制曲线图
plt.show()