TFS文件系统格式化分析

TFS文件系统格式化分析

   TFS文件系统数据服务器 DataServer在使用之前,需要先对文件系统进行格式化,可以对相关的代码进行一下分析.

   (1). 格式化文件系统

//格式化文件系统 int BlockFileManager::format_block_file_system(const SysParam::FileSystemParam& fs_param)         {             int ret = init_super_blk_param(fs_param);             if (TFS_SUCCESS != ret)                 return ret;             ret = create_fs_dir();             if (TFS_SUCCESS != ret)                 return ret;             ret = create_fs_super_blk();             if (TFS_SUCCESS != ret)                 return ret;             ret = create_block(C_MAIN_BLOCK);             if (TFS_SUCCESS != ret)                 return ret;             ret = create_block(C_EXT_BLOCK);             if (TFS_SUCCESS != ret)                 return ret;             return TFS_SUCCESS;         }

  （2）.初始化超级块

//根据文件参数初始化超级块         int BlockFileManager::init_super_blk_param(const SysParam::FileSystemParam& fs_param)         {             memset((void *) &super_block_, 0, sizeof(SuperBlock));             //"TAOBAO"             memcpy(super_block_.mount_tag_, DEV_TAG, sizeof(super_block_.mount_tag_));             super_block_.time_ = time(NULL);             //每个dataserver进程的挂载目录前缀（实际挂载目录类似于/data/tfs1，启动的时候传入序号）             strncpy(super_block_.mount_point_, fs_param.mount_name_.c_str(), MAX_DEV_NAME_LEN);             TBSYS_LOG(INFO, "super block mount point: %s.", super_block_.mount_point_);             int32_t scale = 1024;             //mount_maxsize = 209715200//挂载点的可用空间（以K为单位，例如即是G）             //这里换算为字节             super_block_.mount_point_use_space_ = fs_param.max_mount_size_ * scale;             super_block_.base_fs_type_ = static_cast<BaseFsType> (fs_param.base_fs_type_);             if (EXT4 != super_block_.base_fs_type_ && EXT3_FULL != super_block_.base_fs_type_ && EXT3_FTRUN                 != super_block_.base_fs_type_)             {                 TBSYS_LOG(ERROR, "base fs type is not supported. base fs type: %d", super_block_.base_fs_type_);                 return TFS_ERROR;             }             //超级块存储的偏移量默认值是             super_block_.superblock_reserve_offset_ = fs_param.super_block_reserve_offset_;             //位图起始偏移量=0+2*超级块大小+一个整形的size             super_block_.bitmap_start_offset_ = super_block_.superblock_reserve_offset_ + 2 * sizeof(SuperBlock)+ sizeof(int32_t);             //超级块段的平均尺寸(平均文件大小),默认是K             super_block_.avg_segment_size_ = fs_param.avg_segment_size_;             //block_type_ratio_默认值是,主块/扩展块             super_block_.block_type_ratio_ = fs_param.block_type_ratio_;             //INDEXFILE_SAFE_MULT=4   META_INFO_SIZE=sizeof(MetaInfo)             int32_t data_ratio = super_block_.avg_segment_size_ / (META_INFO_SIZE * INDEXFILE_SAFE_MULT);             //可以得到的数据空间:修正该avail_data_space值大小使之成为data_ratio的整数倍             int64_t avail_data_space = static_cast<int64_t> (super_block_.mount_point_use_space_* (static_cast<float> (data_ratio) / static_cast<float> (data_ratio + 1)));             if (avail_data_space <= 0)             {                 TBSYS_LOG(                     ERROR,                     "format filesystem fail. avail data space: %" PRI64_PREFIX "d, avg segment size: %d, single meta size: %u, data ratio: %d/n",                     avail_data_space, super_block_.avg_segment_size_, META_INFO_SIZE, data_ratio);                 return TFS_ERROR;             }             //main_block_size_默认值是:67108864=64M             super_block_.main_block_size_ = fs_param.main_block_size_;             //extend_block_size_默认值是:33554432=32M             super_block_.extend_block_size_ = fs_param.extend_block_size_;             int32_t main_block_count = 0, extend_block_count = 0;             //通过可得到的数据空间得到主块的个数,扩展块的个数             calc_block_count(avail_data_space, main_block_count, extend_block_count);             super_block_.main_block_count_ = main_block_count;             super_block_.extend_block_count_ = extend_block_count;             super_block_.used_block_count_ = 0;             super_block_.used_extend_block_count_ = 0;             super_block_.hash_slot_ratio_ = fs_param.hash_slot_ratio_;             //计算每块的文件个数=(super_block_.main_block_size_+ super_block_.extend_block_size_ /fs_param.block_type_ratio_)/super_block_.avg_segment_size_             int32_t per_block_file_num = static_cast<int32_t> ((super_block_.main_block_size_+ static_cast<float> (super_block_.extend_block_size_) / fs_param.block_type_ratio_)/ super_block_.avg_segment_size_);             //bucket : file =  hash_slot_ratio_             //hash_slot_ratio = 0.5 //每个Block中hash槽和文件个数的比例             int32_t hash_bucket_size = static_cast<int32_t> (super_block_.hash_slot_ratio_* static_cast<float> (per_block_file_num));             //得出槽的内存大小:存放元数据文件的偏移量信息             int32_t hash_bucket_mem_size = hash_bucket_size * sizeof(int32_t);             //源数据大小=sizeof(MetaInfo)*每块中的文件数             int32_t meta_info_size = META_INFO_SIZE * per_block_file_num;             //映射文件大小=槽的内存大小+源数据大小             int32_t need_mmap_size = hash_bucket_mem_size + meta_info_size;             super_block_.hash_slot_size_ = hash_bucket_size;             int32_t sz = getpagesize();  //内存页的大小             int32_t count = need_mmap_size / sz;  //需要得到页的个数             int32_t remainder = need_mmap_size % sz; //剩余             MMapOption mmap_option;             //修正内存文件映射起始大小,使他成为内存页大小的整数倍             mmap_option.first_mmap_size_ = remainder ? (count + 1) * sz : count * sz;             //每次映射大小             mmap_option.per_mmap_size_ = sz;             //INNERFILE_MAX_MULTIPE=30,最大映射文件大小             mmap_option.max_mmap_size_ = mmap_option.first_mmap_size_ * INNERFILE_MAX_MULTIPE;             super_block_.mmap_option_ = mmap_option;             super_block_.version_ = fs_param.file_system_version_;             super_block_.display();             return TFS_SUCCESS; }

   在初始化超级块中调用了计算物理主块,物理扩展块的函数:

void BlockFileManager::calc_block_count(const int64_t avail_data_space, int32_t& main_block_count,             int32_t& ext_block_count)         {             //计算公式     //avail_data_space=super_block_.extend_block_size_*ext_block_count+ext_block_count*super_block_.block_type_ratio_*super_block_.main_block_size_             //扩展块的个数=avail_data_space / (super_block_.block_type_ratio_* super_block_.main_block_size_ + super_block_.extend_block_size_)             ext_block_count = static_cast<int32_t> (static_cast<float> (avail_data_space) / (super_block_.block_type_ratio_* static_cast<float> (super_block_.main_block_size_) + static_cast<float> (super_block_.extend_block_size_)));             //主块的个数=ext_block_count * super_block_.block_type_ratio_             main_block_count = static_cast<int32_t> (static_cast<float> (ext_block_count) * super_block_.block_type_ratio_);             TBSYS_LOG(INFO,                 "cal block count. avail data space: %" PRI64_PREFIX "d, main block count: %d, ext block count: %d",                 avail_data_space, main_block_count, ext_block_count); }

(3). 创建文件系统目录

int BlockFileManager::create_fs_dir()         {             //super_block_.display();             //创建挂载点目录:每个dataserver进程的挂载目录前缀（实际挂载目录类似于/data/tfs1，启动的时候传入序号）             int ret = mkdir(super_block_.mount_point_, DIR_MODE);             if (ret && errno != EEXIST)             {                 TBSYS_LOG(ERROR, "make extend dir error. dir: %s, ret: %d, error: %d, error desc: %s",                     super_block_.mount_point_, ret, errno, strerror(errno));                 return TFS_ERROR;             }             std::string extend_dir = super_block_.mount_point_;             extend_dir += EXTENDBLOCK_DIR_PREFIX;             //创建扩展目录extend_dir=/data/tfs1/extend/             ret = mkdir(extend_dir.c_str(), DIR_MODE);             if (ret)             {                 TBSYS_LOG(ERROR, "make extend dir:%s error. ret: %d, error: %d", extend_dir.c_str(), ret, errno);                 return TFS_ERROR;             }             //创建索引目录index_dir=/data/tfs1/index/             std::string index_dir = super_block_.mount_point_;             index_dir += INDEX_DIR_PREFIX;             ret = mkdir(index_dir.c_str(), DIR_MODE);             if (ret)             {                 TBSYS_LOG(ERROR, "make index dir error. ret: %d, error: %d", ret, errno);                 return TFS_ERROR;             }               return TFS_SUCCESS; }

 (4). 创建文件系统超级块

//创建文件系统超级块         int BlockFileManager::create_fs_super_blk()         {             uint32_t bit_map_size = calc_bitmap_count();             int super_block_file_size = 2 * sizeof(SuperBlock) + sizeof(int32_t) + 4 * bit_map_size;             char* tmp_buffer = new char[super_block_file_size];             memcpy(tmp_buffer, &super_block_, sizeof(SuperBlock));             memcpy(tmp_buffer + sizeof(SuperBlock), &super_block_, sizeof(SuperBlock));             //两倍的* sizeof(SuperBlock)内存后面* bit_map_size + sizeof(int)的内存全部清零,作为写入的内容             memset(tmp_buffer + 2 * sizeof(SuperBlock), 0, 4 * bit_map_size + sizeof(int));             std::string super_block_file = super_block_.mount_point_;             //super_block_file=data/tfs1//fs_super             super_block_file += SUPERBLOCK_NAME;             FileOperation* super_file_op = new FileOperation(super_block_file, O_RDWR | O_CREAT);             //参数:写入缓冲区,写入字节数,写入的偏移量             int ret = super_file_op->pwrite_file(tmp_buffer, super_block_file_size, 0);             if (TFS_SUCCESS != ret)             {                 TBSYS_LOG(ERROR, "write super block file error. ret: %d.", ret);                 return ret;             }             tbsys::gDelete(super_file_op);             tbsys::gDeleteA(tmp_buffer);             return TFS_SUCCESS; }

   在该函数里面,调用了计算位图槽个数的函数:

uint32_t BlockFileManager::calc_bitmap_count() {             //项目个数=超级块个数+扩展块个数+1             uint32_t item_count = super_block_.main_block_count_ + super_block_.extend_block_count_ + 1;             BitMap tmp_bit_map(item_count);             //得到槽的个数             uint32_t slot_count = tmp_bit_map.get_slot_count();             TBSYS_LOG(INFO, "cal bitmap count. item count: %u, slot count: %u", item_count, slot_count);             return slot_count; }

   对于BitMap类有如下实现方法:

//分别是x80 1000 0000         //      0x40 0100 0000         //      0x20 0010 0000         //      0x10 0001 0000         //      0x08 0000 1000         //      0x04 0000 0100         //      0x02 0000 0010         //      0x01 0000 0001 const unsigned char BitMap::BITMAPMASK[SLOT_SIZE] = {             0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01 };

//set_flag参数默认是false BitMap::BitMap(const uint32_t item_count, const bool set_flag) {             assert(item_count != 0);             //假定主块是,扩展块是,那么+3+1+8-1=20 在右移位 0001 0100===>0000 0010             //item_count=3+9+1=13             //槽的个数=2             slot_count_ = (item_count + SLOT_SIZE - 1) >> 3;              data_ = new char[sizeof(char) * slot_count_]; //分配两个字节缓冲             item_count_ = item_count;             memset(data_, set_flag ? 0xFF : 0x0, slot_count_ * sizeof(char));             set_count_ = set_flag ? item_count : 0;             mount_ = false; }

void BitMap::copy(const uint32_t slot_count, const char* bitmap_data) {             assert(NULL != data_);             assert(slot_count_ == slot_count);             //拷贝两个字节             memcpy(data_, bitmap_data, slot_count);             set_count_ = 0;             //假定主块是,扩展块是,那么+9+1就是item_count_=13             for (uint32_t pos = 0; pos < item_count_; ++pos)             {                 if (test(pos))                 {                     ++set_count_;  //已经占用的个数                 }             }             return; }

//测试该index是否被占用 bool BitMap::test(const uint32_t index) const {     //假定主块是9,扩展块是3,那么+9+1=13=item_count_,而SLOT_SIZE=8,而index在到之间             assert(index < item_count_);             uint32_t quot = index / SLOT_SIZE; //只有可能是,1             uint32_t rem = index % SLOT_SIZE;  //可能是到             return (data_[quot] & BITMAPMASK[rem]) != 0;  //&位操作,两个表达式对应位全为时,结果才为,否则为 }

//设置该index是被占用 void BitMap::set(const uint32_t index) {             assert(index < item_count_);             uint32_t quot = index / SLOT_SIZE;             uint32_t rem = index % SLOT_SIZE;             if (!(data_[quot] & BITMAPMASK[rem]))             {                 //一个字节位,设置该位的值为BITMAPMASK[rem]                 data_[quot] |= BITMAPMASK[rem]; //只要一个位为则结果为                 ++set_count_;             } }

//撤销该index的占用 void BitMap::reset(const uint32_t index) {             assert(index < item_count_);             uint32_t quot = index / SLOT_SIZE;             uint32_t rem = index % SLOT_SIZE;             if (data_[quot] & BITMAPMASK[rem])             {                 data_[quot] &= ~BITMAPMASK[rem];  //~取反,顾名思义，变为，变为                 --set_count_;             } }

(5). 格式化文件系统时创建块

//格式化文件系统时创建块         int BlockFileManager::create_block(const BlockType block_type)         {             int32_t prefix_size = sizeof(BlockPrefix);             char* block_prefix = new char[prefix_size];             memset(block_prefix, 0, prefix_size);             FileFormater* file_formater = NULL;             if (EXT4 == super_block_.base_fs_type_)             {                 file_formater = new Ext4FileFormater();             }             else if (EXT3_FULL == super_block_.base_fs_type_)             {                 file_formater = new Ext3FullFileFormater();             }             else if (EXT3_FTRUN == super_block_.base_fs_type_)             {                 file_formater = new Ext3SimpleFileFormater();             }             else             {                 TBSYS_LOG(ERROR, "base fs type is not supported. base fs type: %d", super_block_.base_fs_type_);                 return TFS_ERROR;             }             int32_t block_count = 0;             int32_t block_size = 0;             if (C_MAIN_BLOCK == block_type)             {                 block_count = super_block_.main_block_count_;                 block_size = super_block_.main_block_size_;             }             else if (C_EXT_BLOCK == block_type)             {                 block_count = super_block_.extend_block_count_;                 block_size = super_block_.extend_block_size_;             }             else             {                 return TFS_ERROR;             }             for (int32_t i = 1; i <= block_count; ++i)             {                 std::string block_file;                 std::stringstream tmp_stream;                 if (C_MAIN_BLOCK == block_type)                 {                     //MAINBLOCK_DIR_PREFIX="/"                     tmp_stream << super_block_.mount_point_ << MAINBLOCK_DIR_PREFIX << i;                 }                 else                 {                     //EXTENDBLOCK_DIR_PREFIX=/extend/                     tmp_stream << super_block_.mount_point_ << EXTENDBLOCK_DIR_PREFIX << (i + super_block_.main_block_count_);                 }                 //格式化时创建块                 //主块=         /data/tfs1/1                  /data/tfs1/2                   /data/tfs1/3                 //对应扩展块   /data/tfs1/extend/1+主块个数 /data/tfs1/extend/2+主块个数  /data/tfs1/extend/3+主块个数                 tmp_stream >> block_file;                 FileOperation* file_op = new FileOperation(block_file, O_RDWR | O_CREAT);                 int ret = file_op->open_file();                 if (ret < 0)                 {                     TBSYS_LOG(ERROR, "allocate space error. ret: %d, error: %d, error desc: %s/n", ret, errno, strerror(errno));                     return ret;                 }                 ret = file_formater->block_file_format(file_op->get_fd(), block_size);                 if (TFS_SUCCESS != ret)                 {                     TBSYS_LOG(ERROR, "allocate space error. ret: %d, error: %d, error desc: %s/n", ret, errno, strerror(errno));                     return ret;                 }                 //格式化块文件后在先写入块前缀                 ret = file_op->pwrite_file(block_prefix, prefix_size, 0);                 if (TFS_SUCCESS != ret)                 {                     TBSYS_LOG(ERROR, "write block file error. physcial block id: %d, block type: %d, ret: %d.", i, block_type,                         ret);                     return ret;                 }                 tbsys::gDelete(file_op);             }             tbsys::gDelete(file_formater);             return TFS_SUCCESS;         }