文件映射读写大文件

近来因工作需要写了个用文件映射方式读写大文件的类，之前以为应该不难，写完后才发现自己火候未到；主要体现在逻辑思维和编程思想方面的不足。写完测试时发现该类在写入数据量方面受限于内存上限，百般寻觅就是未发现问题所在，有心的朋友若发现问题所在请务必告诉我；因此特将源码挂上，以为后人鉴戒！

1，对一个文件读写之前应调用OpenFileOFMapp()得到相关参数。

2，写文件时分了循环调用和一次调用的情况。

3，读文件时判断各种情况

4，关闭文件。

 

////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

 #ifndef _FILEMAPPING_ #define _FILEMAPPING_ #include <windows.h> #include <stdio.h> #include <bitset> #include <string> #include <iostream> #include <time.h> #include <tchar.h> using namespace std; class CFileMapping { public: CFileMapping(); ~CFileMapping(); BOOL OpenFileOFMapp(LPCTSTR lpFileName, BOOL bReadOrWrite = false); //打开或创建文件映射 BOOL ReadFileOFMapp(const CHAR* pcReadData, ULONGLONG llReadOffSet, ULONGLONG lReadLength);//从文件映射中读一块数据 BOOL WriteFileOnce(const CHAR* pcWriteData, ULONG iDataLength);//向文件写一次数据 BOOL WriteFileMore(const CHAR* pcWriteData, ULONG iDataLength);//循环调用写入数据 VOID InitialVariable(); private: /************************************************************************/ /* 在循环调用过程中一些控制变量应当设置为成员变量，这样才能为下一次的调 *//* 用保存一些效信息*/ LONGLONG m_lLeftViewSpace; //视图现能写数据 ULONGLONG m_lNotWroteData; //还未写数据 ULONGLONG m_lActualWriteData; //实际要写入 ULONGLONG m_llWriteBufOffSet; //写地址偏移 ULONGLONG m_llIntervalOffSet; //偏移与实际的差 ULONGLONG m_lHaveWroteView; //视图已写 /************************************************************************/ public: VOID CloseFileOFMapp(); //清理相关变量 BOOL SetMapSize(ULONGLONG YourLength = 0xa00000);//用户可自定义要写时文件大小 BOOL SetViewSize(ULONGLONG YourLength = 16);//用户可指定每次操作的数据大小 private: HANDLE m_hFile;//文件句柄 HANDLE m_hFileMapping; //文件映射句柄 LPVOID m_lpWriteBuf; //写映射区域起始 LPVOID m_lpReadBuf; //读映射区域起始 ULONGLONG m_llFileWriteOffSet;//写偏移 ULONGLONG m_llFileReadOffSet;//读偏移 ULONGLONG m_llMapViewSize;//映射视图大小 ULONGLONG m_llFileMapSize; //映射的大小 ULONGLONG m_llFileSize; //当前文件大小 ULONGLONG m_llVirtualFileSize;//读文件时扩大一点，凑成内存粒度的整数倍 ULONGLONG m_lHaveWroteDataToFile; //写数据大小 DWORD m_dwSysGran; //内存粒度 }; #endif  

 #include "CFileMapping.h" CFileMapping::CFileMapping() { InitialVariable(); //Initial Write Function Variable SYSTEM_INFO SystemInfo; GetSystemInfo(&SystemInfo); m_dwSysGran = SystemInfo.dwAllocationGranularity; m_llMapViewSize = m_dwSysGran * 320; //View size m_llFileMapSize = 0x80000000;//File size } VOID CFileMapping::InitialVariable() { m_hFile = INVALID_HANDLE_VALUE; m_hFileMapping = INVALID_HANDLE_VALUE; m_lpReadBuf = NULL; m_llFileSize = 0; m_llFileReadOffSet = 0; m_llVirtualFileSize = 0; m_lHaveWroteDataToFile = 0; m_llFileWriteOffSet = 0; m_lpWriteBuf = NULL; m_lLeftViewSpace = 0; m_lNotWroteData = 0; m_lActualWriteData = 0; m_llWriteBufOffSet = 0; m_llIntervalOffSet = 0; m_lHaveWroteView = 0; } CFileMapping::~CFileMapping() { InitialVariable(); } BOOL CFileMapping::OpenFileOFMapp(LPCTSTR lpFileName, BOOL bReadOrWrite) { if ((m_hFile = CreateFile(lpFileName, GENERIC_READ|GENERIC_WRITE, 0, NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL)) == INVALID_HANDLE_VALUE)// 创建或打开文件 { return FALSE; } m_llFileSize = GetFileSize(m_hFile, 0); if (!bReadOrWrite) { //写视图偏移量,须为m_dwSysGran的倍数，会比文件长度小,记录两者之间的差值 m_llFileWriteOffSet = (m_llFileSize / m_dwSysGran)*m_dwSysGran; m_llIntervalOffSet = m_llFileSize - m_llFileWriteOffSet; } if ((bReadOrWrite == false)&&(m_llFileSize < m_llFileMapSize))//写入文件时检查文件是否已是最大长度 { if (!(m_hFileMapping = CreateFileMapping(m_hFile, NULL, PAGE_READWRITE, (DWORD)(m_llFileMapSize>>32), (DWORD)(m_llFileMapSize&0xFFFFFFFF), NULL))) { CloseHandle(m_hFile); m_hFile = INVALID_HANDLE_VALUE; return FALSE; } return TRUE; } if (bReadOrWrite)//读取文件 { if ((m_llFileSize % m_dwSysGran) !=0) { m_llVirtualFileSize = (m_dwSysGran - (m_llFileSize % m_dwSysGran)) + m_llFileSize; } if (!(m_hFileMapping = CreateFileMapping(m_hFile, NULL, PAGE_READWRITE, (DWORD)(m_llVirtualFileSize>>32), (DWORD)(m_llVirtualFileSize&0xFFFFFFFF), NULL)))//将整个文件映射,并加长文件 { CloseHandle(m_hFile); m_hFile = INVALID_HANDLE_VALUE; return FALSE; } return TRUE; } return FALSE; } BOOL CFileMapping::WriteFileOnce(const CHAR* pcWriteData, ULONG lDataNeedLength)//将pcWriteData写入文件 { if (m_llFileSize < m_llFileMapSize)//当文件已经达到指定的大小时不写，此时文件已满 { LONGLONG lDataCanWrite = 0; //能写数据 ULONGLONG lDataNotWoite = 0; //未写数据 ULONGLONG lActualWrite = 0; //实际写入 ULONGLONG llFirstOffSet = m_llFileWriteOffSet;//起始偏移 ULONGLONG llWriteBufOffSet = 0; ULONGLONG llIntervalOffSet = 0; ULONGLONG lHaveWroteView = 0; ULONGLONG lSourceDataHaveWrote = 0; BOOL bViewIsFull = false; do{ if (m_lpWriteBuf == NULL)//创建文件映射视图 { if (!(m_lpWriteBuf = MapViewOfFile(m_hFileMapping, FILE_MAP_WRITE, DWORD(m_llFileWriteOffSet>>32), DWORD(m_llFileWriteOffSet&0xFFFFFFFF), m_llMapViewSize))) {//注意当偏移量随着写文件的不断增大，偏移量与文件映射内核对象末尾之间的大小比要分配的视图小时，函数失败 CloseHandle(m_hFileMapping); m_hFileMapping = INVALID_HANDLE_VALUE; return FALSE; } llIntervalOffSet = 0; lHaveWroteView = 0; llWriteBufOffSet = 0; } if ((m_llFileSize > 0)&&(bViewIsFull == false)) { llIntervalOffSet = m_llFileSize - llFirstOffSet;//偏移量与文件末尾的距离 } if (m_llFileSize == 0) { llIntervalOffSet = 0; } lDataCanWrite = LONG(m_llMapViewSize - lHaveWroteView - llIntervalOffSet); if (lDataCanWrite <= 0)//目前无空间 { //写入数据超过视图大小，重新建立新视图 UnmapViewOfFile(m_lpWriteBuf); m_lpWriteBuf = NULL; LONGLONG lViewLeft = m_llFileMapSize - m_llFileWriteOffSet; if (lViewLeft < m_llMapViewSize) {//已经快到文件末尾，不能再映射一个视图大小 CloseFileOFMapp();//关闭当前打开文件 m_lHaveWroteDataToFile = 0; m_llFileWriteOffSet = 0; //将当前时间设为新文件名,注意更改路径和和后缀 char dateStr [9], timeStr [9], Milliseconds[4]; _strdate(dateStr); _strtime(timeStr); SYSTEMTIME TimeFile; GetLocalTime(&TimeFile); ltoa(TimeFile.wMilliseconds,Milliseconds,10); string strTime = (string)dateStr + (string)timeStr + (string)Milliseconds+".txt/0"; #ifdef UNICODE DWORD llength = MultiByteToWideChar(CP_ACP, 0, strTime.c_str(), -1, NULL, 0); wchar_t *wcStr = new wchar_t[llength]; if(!wcStr)delete []wcStr; MultiByteToWideChar(CP_ACP, 0, strTime.c_str(), -1,wcStr,llength); wstring strFileName = (wstring)wcStr; delete []wcStr; #else string strFileName = strTime; #endif strFileName.replace(2,1,TEXT("-")); strFileName.replace(5,1,TEXT("-")); strFileName.replace(10,1,TEXT("-")); strFileName.replace(13,1,TEXT("-")); strFileName.insert(8,TEXT("-")); strFileName.insert(17,TEXT("-")); strFileName = TEXT("E://")+strFileName; LPCTSTR lpFileName = (LPCTSTR)(LPTSTR)strFileName.c_str(); OpenFileOFMapp(lpFileName, false); } } else { lDataNotWoite = lDataNeedLength - lSourceDataHaveWrote; //未写数据 if (lDataCanWrite >= lDataNotWoite) { lActualWrite = lDataNotWoite; bViewIsFull = false; } else//将剩余空间写满 { lActualWrite = lDataCanWrite; bViewIsFull = true; } // if ((m_lHaveWroteDataToFile + m_llFileSize + lActualWrite) <= m_llFileMapSize)//写文件时判断当前物理文件可否容纳 memcpy((CHAR*)m_lpWriteBuf+llWriteBufOffSet+llIntervalOffSet, (CHAR*)pcWriteData+lSourceDataHaveWrote, lActualWrite); lSourceDataHaveWrote += lActualWrite;//为了移动源数据内存指针 m_lHaveWroteDataToFile += lActualWrite;//为保存文件大小 llWriteBufOffSet += lActualWrite;//为视图偏移 lHaveWroteView += lActualWrite;//记录当前视图已用空间 m_llFileWriteOffSet += m_llMapViewSize;//移动偏移量 } }while (lSourceDataHaveWrote < lDataNeedLength); } return true; } BOOL CFileMapping::WriteFileMore(const CHAR* pcNeedWriteData, ULONG lNeedWroteDataLength) { ULONGLONG lSourceDataHaveWrote = 0; //源数据已写 do{ if (m_lpWriteBuf == NULL)//创建文件映射视图 { if (!(m_lpWriteBuf = MapViewOfFile(m_hFileMapping, FILE_MAP_WRITE, DWORD(m_llFileWriteOffSet>>32), DWORD(m_llFileWriteOffSet&0xFFFFFFFF), m_llMapViewSize))) {//注意当偏移量增大，偏移量与映射对象末尾之间的大小比分配的视图小时，函数失败 CloseHandle(m_hFileMapping); m_hFileMapping = INVALID_HANDLE_VALUE; return FALSE; } m_llWriteBufOffSet = m_lHaveWroteView = 0; } m_lLeftViewSpace = m_llMapViewSize - m_lHaveWroteView - m_llIntervalOffSet;///< 判断View中是否有空间可写 if (m_lLeftViewSpace > 0) { m_lNotWroteData = lNeedWroteDataLength - lSourceDataHaveWrote; //未写数据 m_lActualWriteData = (m_lLeftViewSpace < m_lNotWroteData)?m_lLeftViewSpace:m_lNotWroteData; memcpy((CHAR*)m_lpWriteBuf + (m_llWriteBufOffSet + m_llIntervalOffSet), (CHAR*)pcNeedWriteData + lSourceDataHaveWrote, m_lActualWriteData); lSourceDataHaveWrote += m_lActualWriteData;//为了移动源数据内存指针 m_lHaveWroteDataToFile += m_lActualWriteData;//为保存文件大小 m_llWriteBufOffSet += m_lActualWriteData;//为视图偏移 m_lHaveWroteView += m_lActualWriteData;//记录当前视图已用空间 } else { // FlushViewOfFile(m_lpWriteBuf, m_lHaveWroteView); UnmapViewOfFile(m_lpWriteBuf); m_lpWriteBuf = NULL; m_llIntervalOffSet = 0; m_llFileWriteOffSet += m_llMapViewSize;//移动偏移量 //已经到文件末尾，不能再映射一个视图大小 if ((m_llFileMapSize - m_llFileWriteOffSet) < m_llMapViewSize) { CloseFileOFMapp(); m_lHaveWroteDataToFile = 0; m_llFileWriteOffSet = 0; CHAR dateStr [9], timeStr [9], Milliseconds[4]; _strdate(dateStr); _strtime(timeStr); SYSTEMTIME TimeFile; GetLocalTime(&TimeFile); ltoa(TimeFile.wMilliseconds,Milliseconds,10); string strTime = (string)dateStr + (string)timeStr + (string)Milliseconds+".txt/0"; #ifdef UNICODE DWORD llength = MultiByteToWideChar(CP_ACP, 0, strTime.c_str(), -1, NULL, 0); WCHAR *wcStr = new WCHAR[llength]; if(!wcStr)delete []wcStr; MultiByteToWideChar(CP_ACP, 0, strTime.c_str(), -1,wcStr,llength); wstring strFileName = (wstring)wcStr; delete []wcStr; #else string strFileName = strTime; #endif strFileName.replace(2,1,TEXT("-")); strFileName.replace(5,1,TEXT("-")); strFileName.replace(10,1,TEXT("-")); strFileName.replace(13,1,TEXT("-")); strFileName.insert(8,TEXT("-")); strFileName.insert(17,TEXT("-")); strFileName = TEXT("E://")+strFileName; LPCTSTR lpFileName = (LPCTSTR)(LPTSTR)strFileName.c_str(); OpenFileOFMapp(lpFileName, false); } } }while (lSourceDataHaveWrote < lNeedWroteDataLength);//每一次写入的大小判断 return true; } BOOL CFileMapping::ReadFileOFMapp(const CHAR* pcReadData, ULONGLONG llReadOffSet/*起始偏移地址*/, ULONGLONG llReadLength) {//文件有内容时、偏移地址正常、读取长度应大于或等于视图大小 if (m_llFileSize > 0 && llReadLength >= m_llMapViewSize && llReadOffSet >= 0) { ULONGLONG llDataHaveRead = 0; //已读数据的大小 ULONGLONG llNotHaveRead = 0; //未读的数据 ULONGLONG lActualRead = 0; //实际要读的数据 ULONGLONG lRealReadOffSet = 0;//视图偏移量 ULONGLONG lReadIntervalOffSet = 0;//视图起始位置与实际指定位置的间隔 lRealReadOffSet = (llReadOffSet / m_dwSysGran)*m_dwSysGran;//视图映射起始位置 lReadIntervalOffSet = llReadOffSet - lRealReadOffSet; if ((m_llFileSize - llReadOffSet) >= llReadLength)//读取数据块在文件中 { do { if (m_lpReadBuf == NULL)//创建文件映射视图 { if (!(m_lpReadBuf = MapViewOfFile(m_hFileMapping, FILE_MAP_READ, DWORD(lRealReadOffSet>>32), //起始偏移地址须为内存粒度大小的整数倍 DWORD(lRealReadOffSet&0xFFFFFFFF), m_llMapViewSize))) { CloseHandle(m_hFileMapping); m_hFileMapping = INVALID_HANDLE_VALUE; return FALSE; } } llNotHaveRead = llReadLength - llDataHaveRead; if (llNotHaveRead >= m_llMapViewSize) lActualRead = m_llMapViewSize - lReadIntervalOffSet; else lActualRead = llNotHaveRead; memcpy((CHAR*)pcReadData+llDataHaveRead, (CHAR*)m_lpReadBuf+lReadIntervalOffSet, lActualRead); lReadIntervalOffSet = 0; llDataHaveRead += lActualRead; lRealReadOffSet += m_llMapViewSize; UnmapViewOfFile(m_lpReadBuf); m_lpReadBuf = NULL; } while(llDataHaveRead < llReadLength); return true; } //要读取的数据块超出文件，只能从指定位置读到文件末尾 if (( (m_llFileSize - llReadOffSet) > 0)&&( (m_llFileSize - llReadOffSet) < llReadLength)) { do { if (m_lpReadBuf == NULL)//创建文件映射视图 { if (!(m_lpReadBuf = MapViewOfFile(m_hFileMapping, FILE_MAP_READ, DWORD(lRealReadOffSet>>32), //起始偏移地址须为内存粒度大小的整数倍 DWORD(lRealReadOffSet&0xFFFFFFFF), m_llMapViewSize))) { CloseHandle(m_hFileMapping); m_hFileMapping = INVALID_HANDLE_VALUE; return FALSE; } } llNotHaveRead = m_llFileSize - llReadOffSet - llDataHaveRead; if (llNotHaveRead > m_llMapViewSize) lActualRead = m_llMapViewSize - lReadIntervalOffSet; else lActualRead = llNotHaveRead; ::memcpy((CHAR*)pcReadData+llDataHaveRead, (CHAR*)m_lpReadBuf+lReadIntervalOffSet, lActualRead); lReadIntervalOffSet = 0; llDataHaveRead += lActualRead; lRealReadOffSet += m_llMapViewSize; UnmapViewOfFile(m_lpReadBuf); m_lpReadBuf = NULL; } while(llDataHaveRead < m_llFileSize - llReadOffSet); return true; } } return false; } VOID CFileMapping::CloseFileOFMapp() { if (m_lpWriteBuf != NULL) { UnmapViewOfFile(m_lpWriteBuf); // 解锁,释放内存 m_lpWriteBuf = NULL; } if (m_lpReadBuf != NULL) { UnmapViewOfFile(m_lpReadBuf); m_lpReadBuf = NULL; } if (m_hFileMapping != INVALID_HANDLE_VALUE) { CloseHandle(m_hFileMapping);// 关闭映射 m_hFileMapping = INVALID_HANDLE_VALUE; } if (m_hFile != INVALID_HANDLE_VALUE)//将文件长度设为实际长度 { LARGE_INTEGER lDistanceLength; lDistanceLength.QuadPart = m_lHaveWroteDataToFile + m_llFileSize; SetFilePointer(m_hFile, lDistanceLength.LowPart, &lDistanceLength.HighPart, FILE_BEGIN); SetEndOfFile(m_hFile); CloseHandle(m_hFile); m_hFile = INVALID_HANDLE_VALUE; } } BOOL CFileMapping::SetMapSize(ULONGLONG SetMapSize)//文件尽量是个大文件 { if (SetMapSize > 0) { m_llFileMapSize = SetMapSize; return true; } else return false; } BOOL CFileMapping::SetViewSize(ULONGLONG SetViewSize)//视图大小应为内存粒度的整数倍 { if (SetViewSize > 0) { m_llMapViewSize = m_dwSysGran * SetViewSize; return true; } else return false; }  

 #include "CFileMapping.h" void main() { DWORD dwStart=GetTickCount(); CFileMapping FileObject; FileObject.OpenFileOFMapp(TEXT("E:\\07-27-10.txt"), false); ULONG lengthstr = 4 * 1024*1024; UCHAR* pbyBuf = new UCHAR[lengthstr]; for (ULONG icounter = 0; icounter < 500; ++icounter) { FileObject.WriteFileMore((char*)pbyBuf, lengthstr); } FileObject.CloseFileOFMapp(); delete[] pbyBuf; dwStart=GetTickCount()-dwStart; dwStart=0; /* CFileMapping FileObject; BOOL bWriteStr = false; FileObject.OpenFileOFMapp(TEXT("E://07-27-10.txt"), false); string strTest; for (LONG i = 0;i <= 1000000; i++) { if (bWriteStr) strTest += "123457890"; else strTest += "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; } strTest += "///EndOfFilePosition///"; ULONG lengthstr = strlen(strTest.c_str()); FileObject.WriteFileOnce(strTest.c_str(),lengthstr); FileObject.CloseFileOFMapp(); */ // getchar(); }