Google python logpuzzle.py (python 2.7)-优快云博客

本文介绍如何从Apache日志文件中提取特定模式的URL，即所谓的谜题URL，并按照步骤下载对应的图像文件。同时，文章提供了一个Python脚本实现这一过程，包括读取日志文件、解析URL、下载图片以及创建HTML索引页面，帮助用户自动化完成任务。

#!/usr/bin/python
import os
import re
import sys
import urllib

"""Logpuzzle exercise
Given an apache logfile, find the puzzle urls and download the images.

Here's what a puzzle url looks like:
10.254.254.28 - - [06/Aug/2007:00:13:48 -0700] "GET /~foo/puzzle-bar-aaab.jpg HTTP/1.0" 302 528 "-" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6"
"""

[color=red]def read_urls(filename):
"""Returns a list of the puzzle urls from the given log file,
extracting the hostname from the filename itself.
Screens out duplicate urls and returns the urls sorted into
increasing order."""
hostname=os.path.basename(filename)[os.path.basename(filename).index('_')+1:]
cont=open(filename).read()
all_urls=re.findall(r'GET (\S+)',cont)
urls=[]
for url in all_urls:
if 'images/puzzle' in url and 'http://'+hostname+'/'+url not in urls:
urls.append('http://'+hostname+'/'+url)

return sorted(urls)

def download_images(img_urls, dest_dir):
"""Given the urls already in the correct order, downloads
each image into the given directory.
Gives the images local filenames img0, img1, and so on.
Creates an index.html in the directory
with an img tag to show each local image file.
Creates the directory if necessary.
"""
f=open(os.path.join(dest_dir,'index.html'),'w')
f.write('<html><body>')
temp=0
for url in img_urls:
image=urllib.urlretrieve(url,os.path.join(dest_dir,'img'+str(temp)+'.jpg'))
f.write('<img src='+os.path.join(dest_dir,'img'+str(temp))+'.jpg>')
temp=temp+1
f.write('</body></html>')
f.close()
[/color]
def main():
args = sys.argv[1:]

if not args:
print 'usage: [--todir dir] logfile '
sys.exit(1)

todir = ''
if args[0] == '--todir':
todir = args[1]
del args[0:2]

img_urls = read_urls(args[0])

if todir:
download_images(img_urls, todir)
else:
print '\n'.join(img_urls)

if __name__ == '__main__':
main()