Server Develop (六) Linux epoll总结

 Linux  epoll

  epoll是Kernel 2.6后新加入的事件机制,在高并发条件下,远优于select。epoll最大的好处在于它不会随着监听fd数目的增长而降低效率。因为在内核中的select实现中,它是采用轮询来处理的,轮询的fd数目越多,自然耗时越多。并且,在linux/posix_types.h头文件有这样的声明: 

#define __FD_SETSIZE    1024 //select最多同时监听1024个fd

  当然,可以通过修改头文件再重编译内核来扩大这个数目,但这似乎并不治本。

  所以在Nginx中采用了epoll来实现其高并发特性。 

工作方式

  LT(level triggered):水平触发,缺省方式,同时支持block和no-block socket,在这种做法中,内核告诉我们一个文件描述符是否被就绪了,如果就绪了,你就可以对这个就绪的fd进行IO操作。如果你不作任何操作,内核还是会继续通知你的,所以,这种模式编程出错的可能性较小。传统的select\poll都是这种模型的代表。

  ET(edge-triggered):边沿触发,高速工作方式,只支持no-block socket。在这种模式下,当描述符从未就绪变为就绪状态时,内核通过epoll告诉你。然后它会假设你知道文件描述符已经就绪,并且不会再为那个描述符发送更多的就绪通知,直到你做了某些操作导致那个文件描述符不再为就绪状态了(比如:你在发送、接受或者接受请求,或者发送接受的数据少于一定量时导致了一个EWOULDBLOCK错误)。但是请注意,如果一直不对这个fs做IO操作(从而导致它再次变成未就绪状态),内核不会发送更多的通知。

  区别:LT事件不会丢弃,而是只要读buffer里面有数据可以让用户读取,则不断的通知你。而ET则只在事件发生之时通知。

主要的数据结构

   epoll_event的结构如下:

typedef union epoll_data {
        void *ptr;
         int fd;
         __uint32_t u32;
         __uint64_t u64;
} epoll_data_t;//保存触发事件的某个文件描述符相关的数据

struct epoll_event {
         __uint32_t events;      /* epoll event */
         epoll_data_t data;      /* User data variable */
};

  events表示感兴趣的事件和被触发的事件,可能的取值为:

EPOLLIN对应的文件描述符可以读
EPOLLOUT对应的文件描述符可以写
EPOLLPRI对应的文件描述符有紧急的数可读
EPOLLERR对应的文件描述符发生错误
EPOLLHUP对应的文件描述符被挂断
EPOLLET将EPOLL设为边缘触发(Edge Triggered)模式,这是相对于水平触发(Level Triggered)来说的
EPOLLONESHOT只监听一次事件,当监听完这次事件之后,如果还需要继续监听这个socket的话,需要再次把这个socket加入到EPOLL队列里 

 操作函数

    epoll的接口非常简单,用三个相关函数来创建epoll句柄、注册epoll事件以及等待事件的发生。

  创建epoll句柄:

int epoll_create(int size); 
//size表示内核需要监听的数目
//return : epoll文件描述符

  需要注意的是,当创建好epoll句柄后,它就是会占用一个fd值(文件标识符),在linux下如果查看/proc/进程id/fd/,是能够看到这个fd的,所以在使用完epoll后,必须调用close()关闭,否则可能导致fd被耗尽。 

   epoll事件注册函数:

int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event)

//epfd是epoll_create()的返回值
//op表示动作
/* op可被表示为:
    EPOLL_CTL_ADD:注册新的fd到epfd中; 
    EPOLL_CTL_MOD:修改已经注册的fd的监听事件; 
    EPOLL_CTL_DEL:从epfd中删除一个fd; 
*/
//fd是需要监听的fd
//event是内核需要监听的事件

  等待事件发生函数:

int epoll_wait(int epfd, struct epoll_event * events, int maxevents, int timeout)
//epfd是函数返回值
//events是内核监听事件的集合
//maxevents是epoll_wait可以处理的连接事件的最大限度值
//timeout是超时时间

//返回值:请求数

epoll工作流程

   首先,需要调用epoll_create创建epoll,此后我们就可以进行socket/bind/listen,然后调用epoll_ctl进行注册。接下来,就可以通过一个while(1)循环调用epoll_wait来等待事件的发生,然后循环查看接收到的事件并进行处理。如果事件是sever的socketfd我们就要进行accept,并且把接收到client的socketfd加入到要监听的事件中。如果在监听过程中,需要修改操作方式(读/写),可以调用epoll_ctl来重新修改。如果监听到某一个客户端关闭,那么我就需要再次调用epoll_ctl把它从epoll监听事件中删除。

 

 实例

#include <stdio.h>
#include <sys/time.h>
#include <unistd.h>
#include <string.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <sys/epoll.h>
#include <netinet/in.h>
#include <fcntl.h>
#include <stdlib.h>
#include <errno.h>

void setnonblocking(int sockfd)
{
    int opts;
    opts = fcntl(sockfd, F_GETFL);
    if(opts < 0){
        perror("fcntl1 error!\n");
        exit(1);
    }
    opts = opts | O_NONBLOCK;
    if(fcntl(sockfd, F_SETFL, opts) < 0){
        perror("fcntl2 error!\n");
        exit(1);
    }
}

int main()
{
    int fd;
    int on;
    int rs;
    int len;
    int conn;
    char buffer[100];
    int flag1, flag2;
    struct sockaddr_in serv_addr, clt_addr;
    struct timeval timeout;

    int i;
    int nfds;
    int epfd;
    int newfd;

    struct epoll_event ev;
    struct epoll_event events[20];

    fd = socket(AF_INET, SOCK_STREAM, 0);

    on = 1;
    setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
    timeout.tv_sec  = 5;
    timeout.tv_usec = 0;
    setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, (char*)&timeout, sizeof(timeout));

    bzero(&serv_addr, sizeof(struct sockaddr_in));
    serv_addr.sin_family = AF_INET;
    serv_addr.sin_port   = htons(9090);
    serv_addr.sin_addr.s_addr = INADDR_ANY;

    rs = bind(fd, (struct sockaddr*)(&serv_addr), sizeof(struct sockaddr));
    if(rs < 0){
        perror("");
        close(fd);
        return -1;
    }

    setnonblocking(fd);

    epfd = epoll_create(100);

    ev.data.fd = fd;
    ev.events = EPOLLIN|EPOLLET;
    epoll_ctl(epfd, EPOLL_CTL_ADD, fd, &ev);

    rs = listen(fd, 5);
    if(rs < 0){
        perror("");
        close(fd);
        return -1;
    }

    len = sizeof(struct sockaddr);
    
    for(;;){
        nfds = epoll_wait(epfd, events, 20, 500);

        for(i = 0; i < nfds; ++i){
            if(events[i].data.fd == fd){
                conn = accept(fd, (struct sockaddr*)(&clt_addr), (unsigned int*)(&len));
                
                setnonblocking(conn);

                ev.data.fd = conn;
                ev.events = EPOLLIN|EPOLLET;
                epoll_ctl(epfd, EPOLL_CTL_ADD, conn, &ev);
            }
            else if(events[i].events & EPOLLIN){
                if((newfd = events[i].data.fd) < 0)
                    continue;
                bzero(buffer, sizeof(buffer));
                flag1 = recv(newfd, buffer, 100, 0);
                printf("recv: %s\n", buffer);
                printf("recv return: %d\n", flag1);

                ev.data.fd = newfd;
                ev.events = EPOLLOUT|EPOLLET;
                epoll_ctl(epfd, EPOLL_CTL_MOD, newfd, &ev);
            }
            else if(events[i].events & EPOLLOUT){
                if((newfd = events[i].data.fd) < 0)
                    continue;

                bzero(buffer, sizeof(buffer));
                strcpy(buffer, "ACK");
                flag2 = send(newfd, buffer, sizeof(buffer), 0);
                printf("recv return: %d\n\n", flag2);    

                ev.data.fd = newfd;    
                ev.events = EPOLLIN|EPOLLET;
                epoll_ctl(epfd, EPOLL_CTL_MOD, newfd, &ev);
            }
        }
    }
    close(fd);

    return 0;
}

参考

    

        http://www.cnblogs.com/venow/archive/2012/11/30/2790031.html

     http://hi.baidu.com/jingweiyoung/item/ae9fc81714be67dbbf9042b9

     http://www.linuxidc.com/Linux/2011-04/35156p3.htm

   http://www.cppblog.com/converse/archive/2008/04/29/48482.html

D:\software\develop\java\jdk-1.8\jdk-1.8\bin\java.exe "-javaagent:D:\software\develop\ideal\IntelliJ IDEA 2022.1.3\lib\idea_rt.jar=58407:D:\software\develop\ideal\IntelliJ IDEA 2022.1.3\bin" -Dfile.encoding=UTF-8 -classpath D:\software\develop\java\jdk-1.8\jdk-1.8\jre\lib\charsets.jar;D:\software\develop\java\jdk-1.8\jdk-1.8\jre\lib\deploy.jar;D:\software\develop\java\jdk-1.8\jdk-1.8\jre\lib\ext\access-bridge-64.jar;D:\software\develop\java\jdk-1.8\jdk-1.8\jre\lib\ext\cldrdata.jar;D:\software\develop\java\jdk-1.8\jdk-1.8\jre\lib\ext\dnsns.jar;D:\software\develop\java\jdk-1.8\jdk-1.8\jre\lib\ext\jaccess.jar;D:\software\develop\java\jdk-1.8\jdk-1.8\jre\lib\ext\jfxrt.jar;D:\software\develop\java\jdk-1.8\jdk-1.8\jre\lib\ext\localedata.jar;D:\software\develop\java\jdk-1.8\jdk-1.8\jre\lib\ext\nashorn.jar;D:\software\develop\java\jdk-1.8\jdk-1.8\jre\lib\ext\sunec.jar;D:\software\develop\java\jdk-1.8\jdk-1.8\jre\lib\ext\sunjce_provider.jar;D:\software\develop\java\jdk-1.8\jdk-1.8\jre\lib\ext\sunmscapi.jar;D:\software\develop\java\jdk-1.8\jdk-1.8\jre\lib\ext\sunpkcs11.jar;D:\software\develop\java\jdk-1.8\jdk-1.8\jre\lib\ext\zipfs.jar;D:\software\develop\java\jdk-1.8\jdk-1.8\jre\lib\javaws.jar;D:\software\develop\java\jdk-1.8\jdk-1.8\jre\lib\jce.jar;D:\software\develop\java\jdk-1.8\jdk-1.8\jre\lib\jfr.jar;D:\software\develop\java\jdk-1.8\jdk-1.8\jre\lib\jfxswt.jar;D:\software\develop\java\jdk-1.8\jdk-1.8\jre\lib\jsse.jar;D:\software\develop\java\jdk-1.8\jdk-1.8\jre\lib\management-agent.jar;D:\software\develop\java\jdk-1.8\jdk-1.8\jre\lib\plugin.jar;D:\software\develop\java\jdk-1.8\jdk-1.8\jre\lib\resources.jar;D:\software\develop\java\jdk-1.8\jdk-1.8\jre\lib\rt.jar;D:\workspace\Maven\titanic_demo\target\scala-2.12\classes;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\com\clearspring\analytics\stream\2.9.6\stream-2.9.6.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\com\esotericsoftware\kryo-shaded\4.0.2\kryo-shaded-4.0.2.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\com\esotericsoftware\minlog\1.3.0\minlog-1.3.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\com\fasterxml\jackson\core\jackson-annotations\2.12.3\jackson-annotations-2.12.3.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\com\fasterxml\jackson\core\jackson-core\2.12.3\jackson-core-2.12.3.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\com\fasterxml\jackson\core\jackson-databind\2.12.3\jackson-databind-2.12.3.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\com\fasterxml\jackson\module\jackson-module-scala_2.12\2.12.3\jackson-module-scala_2.12-2.12.3.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\com\github\luben\zstd-jni\1.5.0-4\zstd-jni-1.5.0-4.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\com\google\code\findbugs\jsr305\3.0.2\jsr305-3.0.2.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\com\google\code\gson\gson\2.8.6\gson-2.8.6.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\com\google\crypto\tink\tink\1.6.0\tink-1.6.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\com\google\flatbuffers\flatbuffers-java\1.9.0\flatbuffers-java-1.9.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\com\google\guava\guava\16.0.1\guava-16.0.1.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\com\google\protobuf\protobuf-java\3.14.0\protobuf-java-3.14.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\com\ning\compress-lzf\1.0.3\compress-lzf-1.0.3.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\com\thoughtworks\paranamer\paranamer\2.8\paranamer-2.8.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\com\twitter\chill-java\0.10.0\chill-java-0.10.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\com\twitter\chill_2.12\0.10.0\chill_2.12-0.10.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\com\univocity\univocity-parsers\2.9.1\univocity-parsers-2.9.1.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\commons-codec\commons-codec\1.15\commons-codec-1.15.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\commons-collections\commons-collections\3.2.2\commons-collections-3.2.2.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\commons-io\commons-io\2.8.0\commons-io-2.8.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\commons-lang\commons-lang\2.6\commons-lang-2.6.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\commons-logging\commons-logging\1.1.3\commons-logging-1.1.3.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\commons-net\commons-net\3.1\commons-net-3.1.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\io\airlift\aircompressor\0.21\aircompressor-0.21.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\io\dropwizard\metrics\metrics-core\4.2.0\metrics-core-4.2.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\io\dropwizard\metrics\metrics-graphite\4.2.0\metrics-graphite-4.2.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\io\dropwizard\metrics\metrics-jmx\4.2.0\metrics-jmx-4.2.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\io\dropwizard\metrics\metrics-json\4.2.0\metrics-json-4.2.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\io\dropwizard\metrics\metrics-jvm\4.2.0\metrics-jvm-4.2.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\io\netty\netty-all\4.1.68.Final\netty-all-4.1.68.Final.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\io\netty\netty-buffer\4.1.50.Final\netty-buffer-4.1.50.Final.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\io\netty\netty-codec\4.1.50.Final\netty-codec-4.1.50.Final.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\io\netty\netty-common\4.1.50.Final\netty-common-4.1.50.Final.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\io\netty\netty-handler\4.1.50.Final\netty-handler-4.1.50.Final.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\io\netty\netty-resolver\4.1.50.Final\netty-resolver-4.1.50.Final.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\io\netty\netty-transport-native-epoll\4.1.50.Final\netty-transport-native-epoll-4.1.50.Final.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\io\netty\netty-transport-native-unix-common\4.1.50.Final\netty-transport-native-unix-common-4.1.50.Final.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\io\netty\netty-transport\4.1.50.Final\netty-transport-4.1.50.Final.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\jakarta\annotation\jakarta.annotation-api\1.3.5\jakarta.annotation-api-1.3.5.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\jakarta\servlet\jakarta.servlet-api\4.0.3\jakarta.servlet-api-4.0.3.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\jakarta\validation\jakarta.validation-api\2.0.2\jakarta.validation-api-2.0.2.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\jakarta\ws\rs\jakarta.ws.rs-api\2.1.6\jakarta.ws.rs-api-2.1.6.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\javax\activation\activation\1.1.1\activation-1.1.1.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\javax\annotation\javax.annotation-api\1.3.2\javax.annotation-api-1.3.2.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\javax\xml\bind\jaxb-api\2.2.11\jaxb-api-2.2.11.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\log4j\log4j\1.2.17\log4j-1.2.17.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\net\razorvine\pyrolite\4.30\pyrolite-4.30.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\net\sf\py4j\py4j\0.10.9.2\py4j-0.10.9.2.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\antlr\antlr4-runtime\4.8\antlr4-runtime-4.8.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\arrow\arrow-format\2.0.0\arrow-format-2.0.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\arrow\arrow-memory-core\2.0.0\arrow-memory-core-2.0.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\arrow\arrow-memory-netty\2.0.0\arrow-memory-netty-2.0.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\arrow\arrow-vector\2.0.0\arrow-vector-2.0.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\avro\avro-ipc\1.10.2\avro-ipc-1.10.2.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\avro\avro-mapred\1.10.2\avro-mapred-1.10.2.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\avro\avro\1.10.2\avro-1.10.2.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\commons\commons-compress\1.20\commons-compress-1.20.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\commons\commons-crypto\1.1.0\commons-crypto-1.1.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\commons\commons-lang3\3.12.0\commons-lang3-3.12.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\commons\commons-math3\3.4.1\commons-math3-3.4.1.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\commons\commons-text\1.6\commons-text-1.6.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\curator\curator-client\2.13.0\curator-client-2.13.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\curator\curator-framework\2.13.0\curator-framework-2.13.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\curator\curator-recipes\2.13.0\curator-recipes-2.13.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\hadoop\hadoop-client-api\3.3.1\hadoop-client-api-3.3.1.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\hadoop\hadoop-client-runtime\3.3.1\hadoop-client-runtime-3.3.1.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\hive\hive-storage-api\2.7.2\hive-storage-api-2.7.2.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\htrace\htrace-core4\4.1.0-incubating\htrace-core4-4.1.0-incubating.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\ivy\ivy\2.5.0\ivy-2.5.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\orc\orc-core\1.6.11\orc-core-1.6.11.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\orc\orc-mapreduce\1.6.11\orc-mapreduce-1.6.11.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\orc\orc-shims\1.6.11\orc-shims-1.6.11.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\parquet\parquet-column\1.12.1\parquet-column-1.12.1.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\parquet\parquet-common\1.12.1\parquet-common-1.12.1.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\parquet\parquet-encoding\1.12.1\parquet-encoding-1.12.1.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\parquet\parquet-format-structures\1.12.1\parquet-format-structures-1.12.1.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\parquet\parquet-hadoop\1.12.1\parquet-hadoop-1.12.1.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\parquet\parquet-jackson\1.12.1\parquet-jackson-1.12.1.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\spark\spark-catalyst_2.12\3.2.0\spark-catalyst_2.12-3.2.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\spark\spark-core_2.12\3.2.0\spark-core_2.12-3.2.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\spark\spark-kvstore_2.12\3.2.0\spark-kvstore_2.12-3.2.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\spark\spark-launcher_2.12\3.2.0\spark-launcher_2.12-3.2.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\spark\spark-network-common_2.12\3.2.0\spark-network-common_2.12-3.2.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\spark\spark-network-shuffle_2.12\3.2.0\spark-network-shuffle_2.12-3.2.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\spark\spark-sketch_2.12\3.2.0\spark-sketch_2.12-3.2.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\spark\spark-sql_2.12\3.2.0\spark-sql_2.12-3.2.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\spark\spark-tags_2.12\3.2.0\spark-tags_2.12-3.2.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\spark\spark-unsafe_2.12\3.2.0\spark-unsafe_2.12-3.2.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\xbean\xbean-asm9-shaded\4.20\xbean-asm9-shaded-4.20.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\yetus\audience-annotations\0.12.0\audience-annotations-0.12.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\zookeeper\zookeeper-jute\3.6.2\zookeeper-jute-3.6.2.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\apache\zookeeper\zookeeper\3.6.2\zookeeper-3.6.2.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\codehaus\janino\commons-compiler\3.0.16\commons-compiler-3.0.16.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\codehaus\janino\janino\3.0.16\janino-3.0.16.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\fusesource\leveldbjni\leveldbjni-all\1.8\leveldbjni-all-1.8.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\glassfish\hk2\external\aopalliance-repackaged\2.6.1\aopalliance-repackaged-2.6.1.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\glassfish\hk2\external\jakarta.inject\2.6.1\jakarta.inject-2.6.1.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\glassfish\hk2\hk2-api\2.6.1\hk2-api-2.6.1.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\glassfish\hk2\hk2-locator\2.6.1\hk2-locator-2.6.1.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\glassfish\hk2\hk2-utils\2.6.1\hk2-utils-2.6.1.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\glassfish\hk2\osgi-resource-locator\1.0.3\osgi-resource-locator-1.0.3.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\glassfish\jersey\containers\jersey-container-servlet-core\2.34\jersey-container-servlet-core-2.34.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\glassfish\jersey\containers\jersey-container-servlet\2.34\jersey-container-servlet-2.34.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\glassfish\jersey\core\jersey-client\2.34\jersey-client-2.34.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\glassfish\jersey\core\jersey-common\2.34\jersey-common-2.34.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\glassfish\jersey\core\jersey-server\2.34\jersey-server-2.34.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\glassfish\jersey\inject\jersey-hk2\2.34\jersey-hk2-2.34.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\javassist\javassist\3.25.0-GA\javassist-3.25.0-GA.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\jetbrains\annotations\17.0.0\annotations-17.0.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\json4s\json4s-ast_2.12\3.7.0-M11\json4s-ast_2.12-3.7.0-M11.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\json4s\json4s-core_2.12\3.7.0-M11\json4s-core_2.12-3.7.0-M11.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\json4s\json4s-jackson_2.12\3.7.0-M11\json4s-jackson_2.12-3.7.0-M11.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\json4s\json4s-scalap_2.12\3.7.0-M11\json4s-scalap_2.12-3.7.0-M11.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\lz4\lz4-java\1.7.1\lz4-java-1.7.1.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\objenesis\objenesis\2.5.1\objenesis-2.5.1.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\roaringbitmap\RoaringBitmap\0.9.0\RoaringBitmap-0.9.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\roaringbitmap\shims\0.9.0\shims-0.9.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\rocksdb\rocksdbjni\6.20.3\rocksdbjni-6.20.3.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\scala-lang\modules\scala-parser-combinators_2.12\1.1.2\scala-parser-combinators_2.12-1.1.2.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\scala-lang\modules\scala-xml_2.12\1.2.0\scala-xml_2.12-1.2.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\scala-lang\scala-library\2.12.15\scala-library-2.12.15.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\scala-lang\scala-reflect\2.12.15\scala-reflect-2.12.15.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\slf4j\jcl-over-slf4j\1.7.30\jcl-over-slf4j-1.7.30.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\slf4j\jul-to-slf4j\1.7.30\jul-to-slf4j-1.7.30.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\slf4j\slf4j-api\1.7.30\slf4j-api-1.7.30.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\slf4j\slf4j-log4j12\1.7.30\slf4j-log4j12-1.7.30.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\spark-project\spark\unused\1.0.0\unused-1.0.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\threeten\threeten-extra\1.5.0\threeten-extra-1.5.0.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\tukaani\xz\1.8\xz-1.8.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\org\xerial\snappy\snappy-java\1.1.8.4\snappy-java-1.1.8.4.jar;C:\Users\21279\AppData\Local\Coursier\cache\v1\https\repo1.maven.org\maven2\oro\oro\2.0.8\oro-2.0.8.jar Titanic Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties 25/06/19 11:13:36 INFO SparkContext: Running Spark version 3.2.0 25/06/19 11:13:37 INFO ResourceUtils: ============================================================== 25/06/19 11:13:37 INFO ResourceUtils: No custom resources configured for spark.driver. 25/06/19 11:13:37 INFO ResourceUtils: ============================================================== 25/06/19 11:13:37 INFO SparkContext: Submitted application: Titanic 25/06/19 11:13:37 INFO ResourceProfile: Default ResourceProfile created, executor resources: Map(cores -> name: cores, amount: 1, script: , vendor: , memory -> name: memory, amount: 1024, script: , vendor: , offHeap -> name: offHeap, amount: 0, script: , vendor: ), task resources: Map(cpus -> name: cpus, amount: 1.0) 25/06/19 11:13:37 INFO ResourceProfile: Limiting resource is cpu 25/06/19 11:13:37 INFO ResourceProfileManager: Added ResourceProfile id: 0 25/06/19 11:13:37 INFO SecurityManager: Changing view acls to: 21279 25/06/19 11:13:37 INFO SecurityManager: Changing modify acls to: 21279 25/06/19 11:13:37 INFO SecurityManager: Changing view acls groups to: 25/06/19 11:13:37 INFO SecurityManager: Changing modify acls groups to: 25/06/19 11:13:37 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(21279); groups with view permissions: Set(); users with modify permissions: Set(21279); groups with modify permissions: Set() 25/06/19 11:13:38 INFO Utils: Successfully started service 'sparkDriver' on port 58426. 25/06/19 11:13:38 INFO SparkEnv: Registering MapOutputTracker 25/06/19 11:13:38 INFO SparkEnv: Registering BlockManagerMaster 25/06/19 11:13:38 INFO BlockManagerMasterEndpoint: Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information 25/06/19 11:13:38 INFO BlockManagerMasterEndpoint: BlockManagerMasterEndpoint up 25/06/19 11:13:38 INFO SparkEnv: Registering BlockManagerMasterHeartbeat 25/06/19 11:13:39 INFO DiskBlockManager: Created local directory at C:\Users\21279\AppData\Local\Temp\blockmgr-d9717778-5c19-4b54-ba8c-bdc23a2bec0f 25/06/19 11:13:39 INFO MemoryStore: MemoryStore started with capacity 1985.4 MiB 25/06/19 11:13:39 INFO SparkEnv: Registering OutputCommitCoordinator 25/06/19 11:13:39 INFO Utils: Successfully started service 'SparkUI' on port 4040. 25/06/19 11:13:39 INFO SparkUI: Bound SparkUI to 0.0.0.0, and started at http://lxf:4040 25/06/19 11:13:39 INFO Executor: Starting executor ID driver on host lxf 25/06/19 11:13:39 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 58453. 25/06/19 11:13:39 INFO NettyBlockTransferService: Server created on lxf:58453 25/06/19 11:13:39 INFO BlockManager: Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy 25/06/19 11:13:39 INFO BlockManagerMaster: Registering BlockManager BlockManagerId(driver, lxf, 58453, None) 25/06/19 11:13:39 INFO BlockManagerMasterEndpoint: Registering block manager lxf:58453 with 1985.4 MiB RAM, BlockManagerId(driver, lxf, 58453, None) 25/06/19 11:13:39 INFO BlockManagerMaster: Registered BlockManager BlockManagerId(driver, lxf, 58453, None) 25/06/19 11:13:39 INFO BlockManager: Initialized BlockManager: BlockManagerId(driver, lxf, 58453, None) 25/06/19 11:13:39 WARN SparkContext: Using an existing SparkContext; some configuration may not take effect. 25/06/19 11:13:40 INFO SharedState: Setting hive.metastore.warehouse.dir ('null') to the value of spark.sql.warehouse.dir. 25/06/19 11:13:40 INFO SharedState: Warehouse path is 'file:/D:/workspace/Maven/titanic_demo/spark-warehouse'. 25/06/19 11:13:41 WARN FileSystem: Failed to initialize fileystem hdfs://192.168.42:9000/user/hadoop/titanic/titanic.csv: java.io.IOException: Incomplete HDFS URI, no host: hdfs://192.168.42:9000/user/hadoop/titanic/titanic.csv 25/06/19 11:13:41 WARN FileStreamSink: Assume no metadata directory. Error while looking for metadata directory in the path: hdfs://192.168.42:9000/user/hadoop/titanic/titanic.csv. java.io.IOException: Incomplete HDFS URI, no host: hdfs://192.168.42:9000/user/hadoop/titanic/titanic.csv at org.apache.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem.java:183) at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3469) at org.apache.hadoop.fs.FileSystem.access$300(FileSystem.java:174) at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:3574) at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:3521) at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:540) at org.apache.hadoop.fs.Path.getFileSystem(Path.java:365) at org.apache.spark.sql.execution.streaming.FileStreamSink$.hasMetadata(FileStreamSink.scala:53) at org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:370) at org.apache.spark.sql.DataFrameReader.loadV1Source(DataFrameReader.scala:274) at org.apache.spark.sql.DataFrameReader.$anonfun$load$3(DataFrameReader.scala:245) at scala.Option.getOrElse(Option.scala:189) at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:245) at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:188) at Titanic$.main(Titanic.scala:27) at Titanic.main(Titanic.scala) 25/06/19 11:13:41 WARN FileSystem: Failed to initialize fileystem hdfs://192.168.42:9000/user/hadoop/titanic/titanic.csv: java.io.IOException: Incomplete HDFS URI, no host: hdfs://192.168.42:9000/user/hadoop/titanic/titanic.csv Exception in thread "main" java.io.IOException: Incomplete HDFS URI, no host: hdfs://192.168.42:9000/user/hadoop/titanic/titanic.csv at org.apache.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem.java:183) at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3469) at org.apache.hadoop.fs.FileSystem.access$300(FileSystem.java:174) at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:3574) at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:3521) at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:540) at org.apache.hadoop.fs.Path.getFileSystem(Path.java:365) at org.apache.spark.sql.execution.datasources.DataSource$.$anonfun$checkAndGlobPathIfNecessary$1(DataSource.scala:747) at scala.collection.immutable.List.map(List.scala:293) at org.apache.spark.sql.execution.datasources.DataSource$.checkAndGlobPathIfNecessary(DataSource.scala:745) at org.apache.spark.sql.execution.datasources.DataSource.checkAndGlobPathIfNecessary(DataSource.scala:577) at org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:408) at org.apache.spark.sql.DataFrameReader.loadV1Source(DataFrameReader.scala:274) at org.apache.spark.sql.DataFrameReader.$anonfun$load$3(DataFrameReader.scala:245) at scala.Option.getOrElse(Option.scala:189) at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:245) at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:188) at Titanic$.main(Titanic.scala:27) at Titanic.main(Titanic.scala) 25/06/19 11:13:41 INFO SparkContext: Invoking stop() from shutdown hook 25/06/19 11:13:41 INFO SparkUI: Stopped Spark web UI at http://lxf:4040 25/06/19 11:13:41 INFO MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped! 25/06/19 11:13:41 INFO MemoryStore: MemoryStore cleared 25/06/19 11:13:41 INFO BlockManager: BlockManager stopped 25/06/19 11:13:41 INFO BlockManagerMaster: BlockManagerMaster stopped 25/06/19 11:13:41 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: OutputCommitCoordinator stopped! 25/06/19 11:13:41 INFO SparkContext: Successfully stopped SparkContext 25/06/19 11:13:41 INFO ShutdownHookManager: Shutdown hook called 25/06/19 11:13:41 INFO ShutdownHookManager: Deleting directory C:\Users\21279\AppData\Local\Temp\spark-f92de141-507f-4f46-a675-bdf38dd1487f Process finished with exit code 1
最新发布
06-20
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值