kafka协议-Records

介绍

Records是RecordBatch的集合。它遍历Record的过程是，一次遍历每一个RecordBatch的Record。

AbstractRecords

AbstractRecords实现了Records接口，recordsIterator返回 Iterator<Record>实例，来遍历Record。

public abstract class AbstractRecords implements Records {
    
    // 实现Iterable<Record>接口
    private final Iterable<Record> records = new Iterable<Record>() {
        @Override
        public Iterator<Record> iterator() {
            // 调用recordsIterator方法，返回iterator
            return recordsIterator();
        }
    };
    
    // 返回Iterable<Record>
    @Override
    public Iterable<Record> records() {
        return records;
    }
    
    private Iterator<Record> recordsIterator() {
        // 继承AbstractIterator<Record>，实现makeNext方法
        return new AbstractIterator<Record>() {
            // 实现遍历RecordBatch
            private final Iterator<? extends RecordBatch> batches = batches().iterator();
            // 遍历RecordBatch的Record
            private Iterator<Record> records;

            @Override
            protected Record makeNext() {
                // 首先检查records是否还有数据
                if (records != null && records.hasNext())
                    // 如果有，直接返回
                    return records.next();

                // 如果records没有数据，再检查batche是否有数据
                if (batches.hasNext()) {
                    // 如果batches有数据，则更新records为当前batch的数据
                    records = batches.next().iterator();
                    // 递归调用
                    return makeNext();
                }
                // 如果records和batches都没有数据，表明已经读完，则调用allDone方法
                return allDone();
            }
        };
    }

AbstractIterator

AbstractIterator对Iterator的封装，子类只需实现makeNext方法。如果有数据，则赋值next。如果没有，则调用allDone。

public abstract class AbstractIterator<T> implements Iterator<T> {

    private enum State {
        READY, NOT_READY, DONE, FAILED
    }

    private State state = State.NOT_READY;
    private T next;

    @Override
    public boolean hasNext() {
        switch (state) {
            case FAILED:
                throw new IllegalStateException("Iterator is in failed state");
            case DONE:
                return false;
            case READY:
                return true;
            default:
                return maybeComputeNext();
        }
    }

    @Override
    public T next() {
        if (!hasNext())
            throw new NoSuchElementException();
        state = State.NOT_READY;
        if (next == null)
            throw new IllegalStateException("Expected item but none found.");
        return next;
    }

    @Override
    public void remove() {
        throw new UnsupportedOperationException("Removal not supported");
    }

    public T peek() {
        if (!hasNext())
            throw new NoSuchElementException();
        return next;
    }

    // 更新state为DONE
    protected T allDone() {
        state = State.DONE;
        return null;
    }
    // 子类需要实现此方法
    protected abstract T makeNext();

    private Boolean maybeComputeNext() {
        state = State.FAILED;
        next = makeNext();
        if (state == State.DONE) {
            return false;
        } else {
            state = State.READY;
            return true;
        }
    }

FileChannelRecordBatch

FileChannelRecordBatch，从FileChannel读取数据，生成RecordBatch。它可以实例化只有header的RecordBatch，也可以实例化全部数据的RecordBatch。

子类需要实现toMemoryRecordBatch方法，将buffer转换为RecordBatch

    public abstract static class FileChannelRecordBatch extends AbstractRecordBatch {
        protected final long offset;
        protected final byte magic;
        protected final FileChannel channel;
        protected final int position;
        protected final int batchSize;

        private RecordBatch fullBatch;
        private RecordBatch batchHeader;

        FileChannelRecordBatch(long offset,
                               byte magic,
                               FileChannel channel,
                               int position,
                               int batchSize) {
            this.offset = offset;
            this.magic = magic;
            this.channel = channel;
            this.position = position;
            this.batchSize = batchSize;
        }

        @Override
        public CompressionType compressionType() {
            // loadBatchHeader返回RecordBatch，在获取compressionType值
            return loadBatchHeader().compressionType();
        }

        ......

       @Override
        public Iterator<Record> iterator() {
            return loadFullBatch().iterator();
        }

        // 将buffer转换为RecordBatch
        protected abstract RecordBatch toMemoryRecordBatch(ByteBuffer buffer);

        // 返回header部分的长度
        protected abstract int headerSize();

        // 返回RecordBatch，包含完整数据
        protected RecordBatch loadFullBatch() {
            if (fullBatch == null) {
                batchHeader = null;
                fullBatch = loadBatchWithSize(sizeInBytes(), "full record batch");
            }
            return fullBatch;
        }

        //返回只包含header部分的RecordBatch，不包含Record数据
        protected RecordBatch loadBatchHeader() {
            if (fullBatch != null)
                return fullBatch;

            if (batchHeader == null)
                batchHeader = loadBatchWithSize(headerSize(), "record batch header");

            return batchHeader;
        }
        
        // size为要读取batch的长度，从buffer中读取数据，返回RecordBatch
        private RecordBatch loadBatchWithSize(int size, String description) {
            try {
                // 分配buffer
                ByteBuffer buffer = ByteBuffer.allocate(size);
                // 从FileChannel中读取数据，到buffer中
                Utils.readFullyOrFail(channel, buffer, position, description);
                // 设置position为0，准备读
                buffer.rewind();
                // 转换为RecordBatch
                return toMemoryRecordBatch(buffer);
            } catch (IOException e) {
                throw new KafkaException(e);
            }
        }

DefaultFileChannelRecordBatch

DefaultFileChannelRecordBatch实现了FileChannelRecordBatch。toMemoryRecordBatch方法，直接通过buffer实例化DefaultRecordBatch。

    static class DefaultFileChannelRecordBatch extends FileLogInputStream.FileChannelRecordBatch {

        DefaultFileChannelRecordBatch(long offset, byte magic, FileChannel channel,
                                                                          int position,  int batchSize) {
            super(offset, magic, channel, position, batchSize);
        }

        @Override
        protected RecordBatch toMemoryRecordBatch(ByteBuffer buffer) {
            // 实例化DefaultRecordBatch
            return new DefaultRecordBatch(buffer);
        }
        
        // 返回header的长度
        @Override
        protected int headerSize() {
            return RECORD_BATCH_OVERHEAD;
        }
 
    }

RecordBatchIterator

RecordBatchIterator实现了RecordBatch的Iterator接口，提供遍历RecordBatch的方法。

class RecordBatchIterator<T extends RecordBatch> extends AbstractIterator<T> {

    private final LogInputStream<T> logInputStream;

    RecordBatchIterator(LogInputStream<T> logInputStream) {
        this.logInputStream = logInputStream;
    }

    @Override
    protected T makeNext() {
        try {
            // 调用logInputStream的nextBatch方法，获取下一个RecordBatch
            T batch = logInputStream.nextBatch();
            if (batch == null)
                return allDone();
            return batch;
        } catch (IOException e) {
            throw new KafkaException(e);
        }
    }
}

FileRecords

FileRecords实现了AbstractRecords的接口，这里表示从文件中读取数据。这里主要关注batches方法，它实现了遍历RecordBatch

public class FileRecords extends AbstractRecords implements Closeable {
    private final Iterable<FileLogInputStream.FileChannelRecordBatch> batches;

    public FileRecords(File file,
                       FileChannel channel,
                       int start,
                       int end,
                       boolean isSlice) throws IOException {
        this.file = file;
        this.channel = channel;
        this.start = start;
        this.end = end;
        this.isSlice = isSlice;
        this.size = new AtomicInteger();

        if (isSlice) {
            // don't check the file size if this is just a slice view
            size.set(end - start);
        } else {
            int limit = Math.min((int) channel.size(), end);
            size.set(limit - start);

            // if this is not a slice, update the file pointer to the end of the file
            // set the file position to the last byte in the file
            channel.position(limit);
        }
        // 初始化batches
        batches = batchesFrom(start);
    }

    @Override
    public Iterable<FileChannelRecordBatch> batches() {
        return batches;
    }
    
    // 返回Iterable<FileChannelRecordBatch>实例
    private Iterable<FileChannelRecordBatch> batchesFrom(final int start) {
        return new Iterable<FileChannelRecordBatch>() {
            @Override
            public Iterator<FileChannelRecordBatch> iterator() {
                return batchIterator(start);
            }
        };
    }

    private Iterator<FileChannelRecordBatch> batchIterator(int start) {
        final int end;
        if (isSlice)
            end = this.end;
        else
            end = this.sizeInBytes();
        // 实例化FileLogInputStream
        FileLogInputStream inputStream = new FileLogInputStream(channel, start, end);
        // 实例化RecordBatchIterator
        return new RecordBatchIterator<>(inputStream);
    }

LogInputStream

LogInputStream定义了读取下一个batch的方法

interface LogInputStream<T extends RecordBatch> {

    // 读取下一个RecordBatch
    T nextBatch() throws IOException;
}

FileLogInputStream

FileLogInputStream实现了LogInputStream接口，这里重点关注nextBatch方法

public class FileLogInputStream implements LogInputStream<FileLogInputStream.FileChannelRecordBatch> {
    private int position;
    private final int end;
    private final FileChannel channel;
    // HEADER_SIZE_UP_TO_MAGIC为，DefaultRecordBatch中截止到Magic的长度
    private final ByteBuffer logHeaderBuffer = ByteBuffer.allocate(HEADER_SIZE_UP_TO_MAGIC);

    FileLogInputStream(FileChannel channel,
                       int start,
                       int end) {
        this.channel = channel;
        this.position = start;
        this.end = end;
    }

    @Override
    public FileChannelRecordBatch nextBatch() throws IOException {
        if (position + HEADER_SIZE_UP_TO_MAGIC >= end)
            return null;

        logHeaderBuffer.rewind();
        // 读取DefaultRecord前面必要的字段
        Utils.readFullyOrFail(channel, logHeaderBuffer, position, "log header");

        logHeaderBuffer.rewind();
        long offset = logHeaderBuffer.getLong(OFFSET_OFFSET);
        int size = logHeaderBuffer.getInt(SIZE_OFFSET);

        // v0版本的RECORD_OVERHEAD最小，检查是否满足最小长度
        if (size < LegacyRecord.RECORD_OVERHEAD_V0)
            throw new CorruptRecordException(String.format("Record size is smaller than minimum record overhead (%d).", LegacyRecord.RECORD_OVERHEAD_V0));
        // 检查默认版本，是否满足最小长度
        if (position + LOG_OVERHEAD + size > end)
            return null;

        byte magic = logHeaderBuffer.get(MAGIC_OFFSET);
        final FileChannelRecordBatch batch;

        if (magic < RecordBatch.MAGIC_VALUE_V2)
            // 返回旧有的FileChannelRecordBatch
            batch = new LegacyFileChannelRecordBatch(offset, magic, channel, position, size);
        else
            // 返回新的FileChannelRecordBatch
            batch = new DefaultFileChannelRecordBatch(offset, magic, channel, position, size);
        // 更新position的位置，指向下一个RecordBatch
        position += batch.sizeInBytes();
        return batch;
    }

MemoryRecords

MemoryRecords表示从内存中读取数据

public class MemoryRecords extends AbstractRecords {

    private final Iterable<MutableRecordBatch> batches = new Iterable<MutableRecordBatch>() {
        @Override
        public Iterator<MutableRecordBatch> iterator() {
            // 返回RecordBatchIterator，使用ByteBufferLogInputStream实例化
            return new RecordBatchIterator<>(new ByteBufferLogInputStream(buffer.duplicate(), Integer.MAX_VALUE));
        }
    };

    @Override
    public Iterable<MutableRecordBatch> batches() {
        return batches;
    }

ByteBufferLogInputStream

ByteBufferLogInputStream 实现了从ByteBuffer中读取数据，生成Recordbatch.

class ByteBufferLogInputStream implements LogInputStream<MutableRecordBatch> {
    private final ByteBuffer buffer;
    private final int maxMessageSize;

    ByteBufferLogInputStream(ByteBuffer buffer, int maxMessageSize) {
        this.buffer = buffer;
        this.maxMessageSize = maxMessageSize;
    }

    public MutableRecordBatch nextBatch() throws IOException {
        int remaining = buffer.remaining();
        if (remaining < LOG_OVERHEAD)
            return null;
        // 读取 batch的长度
        int recordSize = buffer.getInt(buffer.position() + SIZE_OFFSET);
        // V0 has the smallest overhead, stricter checking is done later
        if (recordSize < LegacyRecord.RECORD_OVERHEAD_V0)
            throw new CorruptRecordException(String.format("Record size is less than the minimum record overhead (%d)", LegacyRecord.RECORD_OVERHEAD_V0));
        if (recordSize > maxMessageSize)
            throw new CorruptRecordException(String.format("Record size exceeds the largest allowable message size (%d).", maxMessageSize));
        // 获取batch的总长度
        int batchSize = recordSize + LOG_OVERHEAD;
        if (remaining < batchSize)
            return null;

        byte magic = buffer.get(buffer.position() + MAGIC_OFFSET);
        // 切片
        ByteBuffer batchSlice = buffer.slice();
        // 设置limit值
        batchSlice.limit(batchSize);
        // buffer设置position指向一下batch
        buffer.position(buffer.position() + batchSize);

        if (magic < 0 || magic > RecordBatch.CURRENT_MAGIC_VALUE)
            throw new CorruptRecordException("Invalid magic found in record: " + magic);

        if (magic > RecordBatch.MAGIC_VALUE_V1)
            return new DefaultRecordBatch(batchSlice);
        else
            return new AbstractLegacyRecordBatch.ByteBufferLegacyRecordBatch(batchSlice);
    }
}

概括

Record是RecordBatch的集合，而Records是RecordBatch的集合。

AbstractRecords实现了从Records遍历Record的方法。

FileRecords继承AbstractRecords，实现了从文件中读取数据。它利用FileLogInputStream，实现了从FileChanne中，读取和解析RecordBatch。

MemoryRecords继承AbstractRecords，实现了从内存中读取数据。它利用ByteBufferLogInputStream，实现了从ByteBuffer中，读取和解析RecordBatch。

转载于:https://my.oschina.net/u/569730/blog/1499590