如何快速读取大文件(看csdn一网友要求写的)没有测试具体的速度。

最新推荐文章于 2023-09-11 16:22:58 发布

weixin_30244889

最新推荐文章于 2023-09-11 16:22:58 发布

阅读量115

点赞数

CC 4.0 BY-SA版权

原文链接：http://www.cnblogs.com/zqonline/archive/2009/11/27/1612231.html

本文介绍了一种使用.NET API高效读取大型文本文件的方法。通过直接内存映射提高读取速度，适用于几百MB大小的文件。该方法对于处理大数据集或大文件的应用场景非常有用。

前言：

只是测试了读取一两百M的文件，感觉还是比较快，好像不能读超过2G以上的文件，我试一下读取3G的文件，没有反应。操作大文件可以参考一下这文

http://support.microsoft.com/kb/189981/zh-cn

下面是我写的一个类，下班了由于时间关系，就没有进行过多的测试：代码如下：

Public Class ReadBigTxt

     Private m_LogPath As String = ""
     Public Sub New ()

     End Sub
     Public Function GetFileTxt( ByVal filename As String ) As String
         Dim str As String = ""
         Try

             Dim vFileHandle As IntPtr = NativeMethods.CreateFileA(filename, & H80000000UI Or & H40000000UI, 0 , IntPtr.Zero, NativeMethods.OPEN_ALWAYS, NativeMethods.FILE_ATTRIBUTE_NORMAL Or _
                         NativeMethods.FILE_FLAG_SEQUENTIAL_SCAN, _
            IntPtr.Zero)
             If NativeMethods.INVALID_HANDLE_VALUE <> CInt (vFileHandle) Then
                 Dim vMappingHandle As IntPtr = NativeMethods.CreateFileMappingA(vFileHandle, IntPtr.Zero, NativeMethods.PAGE_READWRITE, 0 , 0 , " temp " )
                 If vMappingHandle <> IntPtr.Zero Then
                     Dim vHead As IntPtr = NativeMethods.MapViewOfFile(vMappingHandle, NativeMethods.FILE_MAP_COPY Or NativeMethods.FILE_MAP_READ Or NativeMethods.FILE_MAP_WRITE, 0 , 0 , 0 )
                     If vHead <> IntPtr.Zero Then
                         Dim vSize As UInteger = NativeMethods.GetFileSize(vFileHandle, IntPtr.Zero)
                         Dim nb As Byte () = New Byte ( CInt (vSize - 1 )) {}
                        System.Runtime.InteropServices.Marshal.Copy(vHead, nb, 0 , nb.Length)
                         str = System.Text.Encoding.GetEncoding( " GB2312 " ).GetString(nb)
                        nb = Nothing
                        NativeMethods.UnmapViewOfFile(vHead)
                     End If
                    NativeMethods.CloseHandle(vMappingHandle)
                 End If
                NativeMethods.CloseHandle(vFileHandle)
             End If
         Catch ex As Exception
             Throw
         End Try

         Return str ' str.Replace(vbCr, "").Replace(vbLf, "")
     End Function
End Class

< System.Runtime.InteropServices.StructLayoutAttribute(System.Runtime.InteropServices.LayoutKind.Sequential) > _
Public Structure SECURITY_ATTRIBUTES

     ''' PAGE_READWRITE -> 0x04
     Public Const PAGE_READWRITE As Integer = 4

     ''' DWORD->unsigned int
     Public nLength As UInteger

     ''' LPVOID->void*
     Public lpSecurityDescriptor As System.IntPtr

     ''' BOOL->int
     < System.Runtime.InteropServices.MarshalAsAttribute(System.Runtime.InteropServices.UnmanagedType.Bool) > _
     Public bInheritHandle As Boolean
End Structure

Partial Public Class NativeMethods

     Public Const OPEN_ALWAYS As Integer = 4

     Public Const INVALID_HANDLE_VALUE As Integer = - 1

     ''' PAGE_READWRITE -> 0x04
     Public Const PAGE_READWRITE As Integer = 4

     ''' FILE_FLAG_SEQUENTIAL_SCAN -> 0x08000000
     Public Const FILE_FLAG_SEQUENTIAL_SCAN As Integer = 134217728

     ''' GENERIC_WRITE -> (0x40000000L)
     Public Const GENERIC_WRITE As Integer = 1073741824

     ''' FILE_ATTRIBUTE_NORMAL -> 0x00000080
     Public Const FILE_ATTRIBUTE_NORMAL As Integer = 128

     ''' GENERIC_READ -> (0x80000000L)
     Public Const GENERIC_READ As Integer = - 2147483648

     ' FILE_MAP_COPY -> SECTION_QUERY
     Public Const FILE_MAP_COPY As Integer = 1

     ''' SECTION_QUERY -> 0x0001
     Public Const SECTION_QUERY As Integer = 1

     ' FILE_MAP_READ -> SECTION_MAP_READ
     Public Const FILE_MAP_READ As Integer = 4

     ''' SECTION_MAP_READ -> 0x0004
     Public Const SECTION_MAP_READ As Integer = 4

     '   Private Const GENERIC_WRITE = &H40000000
     ' Private Const GENERIC_READ = &H80000000

     ' FILE_MAP_WRITE -> SECTION_MAP_WRITE
     Public Const FILE_MAP_WRITE As Integer = 2

     ''' SECTION_MAP_WRITE -> 0x0002
     Public Const SECTION_MAP_WRITE As Integer = 2

     ''' Return Type: HANDLE->void*
     ''' lpFileName: LPCSTR->CHAR*
     ''' dwDesiredAccess: DWORD->unsigned int
     ''' dwShareMode: DWORD->unsigned int
     ''' lpSecurityAttributes: LPSECURITY_ATTRIBUTES->_SECURITY_ATTRIBUTES*
     ''' dwCreationDisposition: DWORD->unsigned int
     ''' dwFlagsAndAttributes: DWORD->unsigned int
     ''' hTemplateFile: HANDLE->void*
     < System.Runtime.InteropServices.DllImportAttribute( " kernel32.dll " , EntryPoint: = " CreateFileA " ) > _
     Public Shared Function CreateFileA( < System.Runtime.InteropServices.InAttribute(), System.Runtime.InteropServices.MarshalAsAttribute(System.Runtime.InteropServices.UnmanagedType.LPStr) > ByVal lpFileName As String , ByVal dwDesiredAccess As UInteger , ByVal dwShareMode As UInteger , < System.Runtime.InteropServices.InAttribute() > ByVal lpSecurityAttributes As System.IntPtr, ByVal dwCreationDisposition As UInteger , ByVal dwFlagsAndAttributes As UInteger , < System.Runtime.InteropServices.InAttribute() > ByVal hTemplateFile As System.IntPtr) As System.IntPtr
     End Function

     < System.Runtime.InteropServices.DllImportAttribute( " kernel32.dll " , EntryPoint: = " CreateFileW " ) > _
     Public Shared Function CreateFileW( < System.Runtime.InteropServices.InAttribute(), System.Runtime.InteropServices.MarshalAsAttribute(System.Runtime.InteropServices.UnmanagedType.LPWStr) > ByVal lpFileName As String , ByVal dwDesiredAccess As UInteger , ByVal dwShareMode As UInteger , < System.Runtime.InteropServices.InAttribute() > ByVal lpSecurityAttributes As System.IntPtr, ByVal dwCreationDisposition As UInteger , ByVal dwFlagsAndAttributes As UInteger , < System.Runtime.InteropServices.InAttribute() > ByVal hTemplateFile As System.IntPtr) As System.IntPtr
     End Function
     ''' Return Type: BOOL->int
     ''' hObject: HANDLE->void*
     < System.Runtime.InteropServices.DllImportAttribute( " kernel32.dll " , EntryPoint: = " CloseHandle " ) > _
     Public Shared Function CloseHandle( < System.Runtime.InteropServices.InAttribute() > ByVal hObject As System.IntPtr) As < System.Runtime.InteropServices.MarshalAsAttribute(System.Runtime.InteropServices.UnmanagedType.Bool) > Boolean
     End Function

     ''' Return Type: HANDLE->void*
     ''' hFile: HANDLE->void*
     ''' lpFileMappingAttributes: LPSECURITY_ATTRIBUTES->_SECURITY_ATTRIBUTES*
     ''' flProtect: DWORD->unsigned int
     ''' dwMaximumSizeHigh: DWORD->unsigned int
     ''' dwMaximumSizeLow: DWORD->unsigned int
     ''' lpName: LPCSTR->CHAR*
     < System.Runtime.InteropServices.DllImportAttribute( " kernel32.dll " , EntryPoint: = " CreateFileMappingA " ) > _
     Public Shared Function CreateFileMappingA( < System.Runtime.InteropServices.InAttribute() > ByVal hFile As System.IntPtr, < System.Runtime.InteropServices.InAttribute() > ByVal lpFileMappingAttributes As System.IntPtr, ByVal flProtect As UInteger , ByVal dwMaximumSizeHigh As UInteger , ByVal dwMaximumSizeLow As UInteger , < System.Runtime.InteropServices.InAttribute(), System.Runtime.InteropServices.MarshalAsAttribute(System.Runtime.InteropServices.UnmanagedType.LPStr) > ByVal lpName As String ) As System.IntPtr
     End Function

     ''' Return Type: BOOL->int
     ''' lpBaseAddress: LPCVOID->void*
     < System.Runtime.InteropServices.DllImportAttribute( " kernel32.dll " , EntryPoint: = " UnmapViewOfFile " ) > _
     Public Shared Function UnmapViewOfFile( < System.Runtime.InteropServices.InAttribute() > ByVal lpBaseAddress As System.IntPtr) As < System.Runtime.InteropServices.MarshalAsAttribute(System.Runtime.InteropServices.UnmanagedType.Bool) > Boolean
     End Function

     ''' Return Type: LPVOID->void*
     ''' hFileMappingObject: HANDLE->void*
     ''' dwDesiredAccess: DWORD->unsigned int
     ''' dwFileOffsetHigh: DWORD->unsigned int
     ''' dwFileOffsetLow: DWORD->unsigned int
     ''' dwNumberOfBytesToMap: SIZE_T->ULONG_PTR->unsigned int
     < System.Runtime.InteropServices.DllImportAttribute( " kernel32.dll " , EntryPoint: = " MapViewOfFile " ) > _
     Public Shared Function MapViewOfFile( < System.Runtime.InteropServices.InAttribute() > ByVal hFileMappingObject As System.IntPtr, ByVal dwDesiredAccess As UInteger , ByVal dwFileOffsetHigh As UInteger , ByVal dwFileOffsetLow As UInteger , ByVal dwNumberOfBytesToMap As UInteger ) As System.IntPtr
     End Function

     ''' Return Type: DWORD->unsigned int
     ''' hFile: HANDLE->void*
     ''' lpFileSizeHigh: LPDWORD->DWORD*
     < System.Runtime.InteropServices.DllImportAttribute( " kernel32.dll " , EntryPoint: = " GetFileSize " ) > _
     Public Shared Function GetFileSize( < System.Runtime.InteropServices.InAttribute() > ByVal hFile As System.IntPtr, ByVal lpFileSizeHigh As System.IntPtr) As UInteger
     End Function
End Class

代码是根据C#转过来，然后自己在找的相应API的声明写的。

这里在给CreateFile的参数进行给值时提示： 常量表达式无法在类型“UInteger”中表示

如下这段代码：

Dim vFileHandle As IntPtr = NativeMethods.CreateFileA(filename, & H80000000UI Or & H40000000UI, 0 ,

需要在&H8000000加上UI,具体原因，我也不知道，时间关系，先不研究了！

调用：

         Dim r As New ReadBigTxt
         Dim f As New System.Windows.Forms.OpenFileDialog
         If f.ShowDialog = Windows.Forms.DialogResult.OK Then
             Dim t = Now
            Debug.WriteLine(r.GetFileTxt(f.FileName))
             MsgBox (t.ToString & "     " & Now.ToString)
         End If

初步测试了一下，打开一两百M的文件，还是比较快，就几秒种！

转载于:https://www.cnblogs.com/zqonline/archive/2009/11/27/1612231.html