lucene-2.9.0 索引过程(三) 过程简述

最新推荐文章于 2018-05-04 11:28:31 发布

原创最新推荐文章于 2018-05-04 11:28:31 发布 · 2k 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#pointers #byte #string #buffer #文档 #file

IR 专栏收录该内容

16 篇文章

订阅专栏

本文深入探讨了Lucene的索引机制，包括文档添加、索引合并及frq、prx、tii、tis文件的写入过程。详细解释了如何通过delta编码来节省存储空间，并介绍了复合文件cfs的创建方式。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

索引过程
IndexWriter.addDocument(Document) line: 2428
IndexWriter.addDocument(Document, Analyzer) line: 2454
DocumentsWriter.addDocument(Document, Analyzer) line: 750
DocumentsWriter.updateDocument(Document, Analyzer, Term) line: 772
DocFieldProcessorPerThread.processDocument() line: 254
DocInverterPerField.processFields(Fieldable[], int) line: 195
TermsHashPerField.add() line: 391

触发合并和写文档过程
IndexWriter.addDocument(Document) line: 2428
IndexWriter.addDocument(Document, Analyzer) line: 2475
IndexWriter.flush(boolean, boolean, boolean) line: 4166
IndexWriter.doFlush(boolean, boolean) line: 4175
IndexWriter.doFlushInternal(boolean, boolean) line: 4277
DocumentsWriter.flush(boolean) line: 581
DocFieldProcessor.flush(Collection, SegmentWriteState) line: 63
DocInverter.flush(Map, SegmentWriteState) line: 76
TermsHash.flush(Map, SegmentWriteState) line: 145
FreqProxTermsWriter.flush(Map, SegmentWriteState) line: 129

1.frq文件写入 docID(delta值) + tf

FreqProxTermsWriter.flush(Map, SegmentWriteState) line: 129
FreqProxTermsWriter.appendPostings(FreqProxTermsWriterPerField[], FormatPostingsFieldsConsumer) line: 217
FormatPostingsDocsWriter.addDoc(int, int) line: 74

// 值大致是这么写的
final int delta = docID - lastDocID;

lastDocID = docID;
if (omitTermFreqAndPositions) //不记录频率和位置
    out.writeVInt(delta);
else if (1 == termDocFreq)
    out.writeVInt((delta<<1) | 1);
else {
    out.writeVInt(delta<<1);
    out.writeVInt(termDocFreq);
}

2.prx文件写入 position(delta值)
FreqProxTermsWriter.flush(Map, SegmentWriteState) line: 129
FreqProxTermsWriter.appendPostings(FreqProxTermsWriterPerField[], FormatPostingsFieldsConsumer) line: 245
FormatPostingsPositionsWriter.addPosition(int, byte[], int, int) line: 56

3.tii文件写入
// 初始化
TermInfosWriter(Directory directory, String segment, FieldInfos fis,
                  int interval)
       throws IOException {
    initialize(directory, segment, fis, interval, false); // 生成tis
    other = new TermInfosWriter(directory, segment, fis, interval, true);
    other.other = this;
}

FreqProxTermsWriter.flush(Map, SegmentWriteState) line: 149
FormatPostingsFieldsWriter.finish() line: 71
TermInfosWriter.close() line: 220

// 值大致是这么写的
if (!isIndex && size % indexInterval == 0)
   // other.add(lastFieldNumber, lastTermBytes, lastTermBytesLength, lastTi);                      // add an index term
{
   output.writeVInt(start);                      // write shared prefix length
   output.writeVInt(length);                     // write delta length
   output.writeBytes(termBytes, start, length); // write delta bytes
   output.writeVInt(fieldNumber);                // write field num
   output.writeVInt(ti.docFreq);                       // write doc freq
   output.writeVLong(ti.freqPointer - lastTi.freqPointer); // write pointers
   output.writeVLong(ti.proxPointer - lastTi.proxPointer);

if (ti.docFreq >= skipInterval)
{
output.writeVInt(ti.skipOffset);
}

   if (isIndex)
{
     output.writeVLong(other.output.getFilePointer() - lastIndexPointer);
     lastIndexPointer = other.output.getFilePointer(); // write pointer
}

}

4.tis文件写入
// 初始化
TermInfosWriter(Directory directory, String segment, FieldInfos fis,
                  int interval)
       throws IOException {
    initialize(directory, segment, fis, interval, false);
    other = new TermInfosWriter(directory, segment, fis, interval, true); // 生成tii
    other.other = this;
}

FreqProxTermsWriter.appendPostings(FreqProxTermsWriterPerField[], FormatPostingsFieldsConsumer) line: 276
FormatPostingsDocsWriter.finish() line: 113
TermInfosWriter.add(int, byte[], int, TermInfo) line: 169
TermInfosWriter.add(int, byte[], int, TermInfo) line: 171
TermInfosWriter.writeTerm(int, byte[], int) line: 196

值大致是这么写的
output.writeVInt(start);                      // write shared prefix length
output.writeVInt(length);                     // write delta length
output.writeBytes(termBytes, start, length); // write delta bytes
output.writeVInt(fieldNumber);                // write field num
output.writeVInt(ti.docFreq);                       // write doc freq
output.writeVLong(ti.freqPointer - lastTi.freqPointer); // write pointers
output.writeVLong(ti.proxPointer - lastTi.proxPointer);

if (ti.docFreq >= skipInterval)
{
output.writeVInt(ti.skipOffset);
}

if (isIndex)
{
   output.writeVLong(other.output.getFilePointer() - lastIndexPointer);
   lastIndexPointer = other.output.getFilePointer(); // write pointer
}

5.写入复合文件cfs

IndexWriter.addDocument(Document) line: 2428
IndexWriter.addDocument(Document, Analyzer) line: 2475
IndexWriter.flush(boolean, boolean, boolean) line: 4166
IndexWriter.doFlush(boolean, boolean) line: 4175
IndexWriter.doFlushInternal(boolean, boolean) line: 4321
DocumentsWriter.createCompoundFile(String) line: 614
CompoundFileWriter.close() line: 137

合并以下文件
_0.frq
_0.nrm
_0.tii
_0.tis
_0.fnm
_0.prx
为_0.cfs

值大致是这么写的
os.writeVInt(entries.size());
{ // 遍历欲合并文件,写入文件名
os.writeLong(0); // for now
os.writeString(fe.file); // file name
}
{ // 遍历欲合并文件,拷贝数据
copyFile(fe, os, buffer);
}
{ // 遍历欲合并文件,写入文件指针
os.seek(fe.directoryOffset);
os.writeLong(fe.dataOffset);
}