聊聊flink的BlobServer

序

本文主要研究一下flink的BlobServer

BlobServer

flink-1.7.2/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobServer.java

public class BlobServer extends Thread implements BlobService, BlobWriter, PermanentBlobService, TransientBlobService {

    /** The log object used for debugging. */
    private static final Logger LOG = LoggerFactory.getLogger(BlobServer.class);

    /** Counter to generate unique names for temporary files. */
    private final AtomicLong tempFileCounter = new AtomicLong(0);

    /** The server socket listening for incoming connections. */
    private final ServerSocket serverSocket;

    /** Blob Server configuration. */
    private final Configuration blobServiceConfiguration;

    /** Indicates whether a shutdown of server component has been requested. */
    private final AtomicBoolean shutdownRequested = new AtomicBoolean();

    /** Root directory for local file storage. */
    private final File storageDir;

    /** Blob store for distributed file storage, e.g. in HA. */
    private final BlobStore blobStore;

    /** Set of currently running threads. */
    private final Set<BlobServerConnection> activeConnections = new HashSet<>();

    /** The maximum number of concurrent connections. */
    private final int maxConnections;

    /** Lock guarding concurrent file accesses. */
    private final ReadWriteLock readWriteLock;

    /**
     * Shutdown hook thread to ensure deletion of the local storage directory.
     */
    private final Thread shutdownHook;

    // --------------------------------------------------------------------------------------------

    /**
     * Map to store the TTL of each element stored in the local storage, i.e. via one of the {@link
     * #getFile} methods.
     **/
    private final ConcurrentHashMap<Tuple2<JobID, TransientBlobKey>, Long> blobExpiryTimes =
        new ConcurrentHashMap<>();

    /** Time interval (ms) to run the cleanup task; also used as the default TTL. */
    private final long cleanupInterval;

    /**
     * Timer task to execute the cleanup at regular intervals.
     */
    private final Timer cleanupTimer;

    /**
     * Instantiates a new BLOB server and binds it to a free network port.
     *
     * @param config Configuration to be used to instantiate the BlobServer
     * @param blobStore BlobStore to store blobs persistently
     *
     * @throws IOException
     *         thrown if the BLOB server cannot bind to a free network port or if the
     *         (local or distributed) file storage cannot be created or is not usable
     */
    public BlobServer(Configuration config, BlobStore blobStore) throws IOException {
        this.blobServiceConfiguration = checkNotNull(config);
        this.blobStore = checkNotNull(blobStore);
        this.readWriteLock = new ReentrantReadWriteLock();

        // configure and create the storage directory
        this.storageDir = BlobUtils.initLocalStorageDirectory(config);
        LOG.info("Created BLOB server storage directory {}", storageDir);

        // configure the maximum number of concurrent connections
        final int maxConnections = config.getInteger(BlobServerOptions.FETCH_CONCURRENT);
        if (maxConnections >= 1) {
            this.maxConnections = maxConnections;
        }
        else {
            LOG.warn("Invalid value for maximum connections in BLOB server: {}. Using default value of {}",
                    maxConnections, BlobServerOptions.FETCH_CONCURRENT.defaultValue());
            this.maxConnections = BlobServerOptions.FETCH_CONCURRENT.defaultValue();
        }

        // configure the backlog of connections
        int backlog = config.getInteger(BlobServerOptions.FETCH_BACKLOG);
        if (backlog < 1) {
            LOG.warn("Invalid value for BLOB connection backlog: {}. Using default value of {}",
                    backlog, BlobServerOptions.FETCH_BACKLOG.defaultValue());
            backlog = BlobServerOptions.FETCH_BACKLOG.defaultValue();
        }

        // Initializing the clean up task
        this.cleanupTimer = new Timer(true);

        this.cleanupInterval = config.getLong(BlobServerOptions.CLEANUP_INTERVAL) * 1000;
        this.cleanupTimer
            .schedule(new TransientBlobCleanupTask(blobExpiryTimes, readWriteLock.writeLock(),
                storageDir, LOG), cleanupInterval, cleanupInterval);

        this.shutdownHook = ShutdownHookUtil.addShutdownHook(this, getClass().getSimpleName(), LOG);

        //  ----------------------- start the server -------------------

        final String serverPortRange = config.getString(BlobServerOptions.PORT);
        final Iterator<Integer> ports = NetUtils.getPortRangeFro