C code for a simple HTTP client

1. Linux provide a high level socket API that will allow programmer to easily connect to any TCP or UDP services.

   In this tutorial, we will see how this works by implementing a simple HTTP client which will get request a web page given the hostname and the page name, then read the server answer and output the HTML content of the reply.

   To be able to connect to a service built on top of TCP, we first need to create a socket for the TCP protocol, fill in a network address structure representing our destination and the port to connect to and use the latter to connect to the remote server.

   From there, we will be able to send and receive data over the network. Once we are done, we will close the connection.

   Below is the C code for a simple HTTP client that will get the host and the page to request from the command line arguments, resolve the hostname name to an IP, connect to this IP on port 80, build the HTTP query, send it and the retrieve the page content.

3. #include <stdio.h>
4. #include <sys/socket.h>
5. #include <arpa/inet.h>
6. #include <stdlib.h>
7. #include <netdb.h>
8. #include <string.h>
9. int create_tcp_socket();
10. char *get_ip(char *host);
11. char *build_get_query(char *host, char *page);
12. void usage();
13.  
14. #define HOST "coding.debuntu.org"
15. #define PAGE "/"
16. #define PORT 80
17. #define USERAGENT "HTMLGET 1.0"
18.  
19. int main(int argc, char **argv)
20. {
21.   struct sockaddr_in *remote;
22.   int sock;
23.   int tmpres;
24.   char *ip;
25.   char *get;
26.   char buf[BUFSIZ+1];
27.   char *host;
28.   char *page;
29.  
30.   if(argc == 1){
31.     usage();
32.     exit(2);
33.   }  
34.   host = argv[1];
35.   if(argc > 2){
36.     page = argv[2];
37.   }else{
38.     page = PAGE;
39.   }
40.   sock = create_tcp_socket();
41.   ip = get_ip(host);
42.   fprintf(stderr, "IP is %s/n", ip);
43.   remote = (struct sockaddr_in *)malloc(sizeof(struct sockaddr_in *));
44.   remote->sin_family = AF_INET;
45.   tmpres = inet_pton(AF_INET, ip, (void *)(&(remote->sin_addr.s_addr)));
46.   if( tmpres < 0)  
47.   {
48.     perror("Can't set remote->sin_addr.s_addr");
49.     exit(1);
50.   }else if(tmpres == 0)
51.   {
52.     fprintf(stderr, "%s is not a valid IP address/n", ip);
53.     exit(1);
54.   }
55.   remote->sin_port = htons(PORT);
56.  
57.   if(connect(sock, (struct sockaddr *)remote, sizeof(struct sockaddr)) < 0){
58.     perror("Could not connect");
59.     exit(1);
60.   }
61.   get = build_get_query(host, page);
62.   fprintf(stderr, "Query is:/n<<START>>/n%s<<END>>/n", get);
63.  
64.   //Send the query to the server
65.   int sent = 0;
66.   while(sent < strlen(get))
67.   {
68.     tmpres = send(sock, get+sent, strlen(get)-sent, 0);
69.     if(tmpres == -1){
70.       perror("Can't send query");
71.       exit(1);
72.     }
73.     sent += tmpres;
74.   }
75.   //now it is time to receive the page
76.   memset(buf, 0, sizeof(buf));
77.   int htmlstart = 0;
78.   char * htmlcontent;
79.   while((tmpres = recv(sock, buf, BUFSIZ, 0)) > 0){
80.     if(htmlstart == 0)
81.     {
82.       /* Under certain conditions this will not work.
83.       * If the /r/n/r/n part is splitted into two messages
84.       * it will fail to detect the beginning of HTML content
85.       */
86.       htmlcontent = strstr(buf, "/r/n/r/n");
87.       if(htmlcontent != NULL){
88.         htmlstart = 1;
89.         htmlcontent += 4;
90.       }
91.     }else{
92.       htmlcontent = buf;
93.     }
94.     if(htmlstart){
95.       fprintf(stdout, htmlcontent);
96.     }
97.  
98.     memset(buf, 0, tmpres);
99.   }
100.   if(tmpres < 0)
101.   {
102.     perror("Error receiving data");
103.   }
104.   free(get);
105.   free(remote);
106.   free(ip);
107.   close(sock);
108.   return 0;
109. }
110.  
111. void usage()
112. {
113.   fprintf(stderr, "USAGE: htmlget host [page]/n/
114. /thost: the website hostname. ex: coding.debuntu.org/n/
115. /tpage: the page to retrieve. ex: index.html, default: //n");
116. }
117.  
118.  
119. int create_tcp_socket()
120. {
121.   int sock;
122.   if((sock = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0){
123.     perror("Can't create TCP socket");
124.     exit(1);
125.   }
126.   return sock;
127. }
128.  
129.  
130. char *get_ip(char *host)
131. {
132.   struct hostent *hent;
133.   int iplen = 15; //XXX.XXX.XXX.XXX
134.   char *ip = (char *)malloc(iplen+1);
135.   memset(ip, 0, iplen+1);
136.   if((hent = gethostbyname(host)) == NULL)
137.   {
138.     herror("Can't get IP");
139.     exit(1);
140.   }
141.   if(inet_ntop(AF_INET, (void *)hent->h_addr_list[0], ip, iplen) == NULL)
142.   {
143.     perror("Can't resolve host");
144.     exit(1);
145.   }
146.   return ip;
147. }
148.  
149. char *build_get_query(char *host, char *page)
150. {
151.   char *query;
152.   char *getpage = page;
153.   char *tpl = "GET /%s HTTP/1.0/r/nHost: %s/r/nUser-Agent: %s/r/n/r/n";
154.   if(getpage[0] == '/'){
155.     getpage = getpage + 1;
156.     fprintf(stderr,"Removing leading /"//", converting %s to %s/n", page, getpage);
157.   }
158.   // -5 is to consider the %s %s %s in tpl and the ending /0
159.   query = (char *)malloc(strlen(host)+strlen(getpage)+strlen(USERAGENT)+strlen(tpl)-5);
160.   sprintf(query, tpl, getpage, host, USERAGENT);
161.   return query;
162. }

 

To compile it, run:

$ gcc -o htmlget htmlget.c
$ ./htmlget 
USAGE: htmlget host [page]
	host: the website hostname. ex: coding.debuntu.org
	page: the page to retrieve. ex: index.html, default: /

Informative messages and errors are printed to stderr. The content of the page is printed to stdout. Thus, to save the HTML content of a page to a file, you will need to run:
$ ./htmlget coding.debuntu.org category > /tmp/page.html