Hive Job log解析——perl脚本

最新推荐文章于 2021-02-01 15:48:09 发布

最新推荐文章于 2021-02-01 15:48:09 发布 · 252 阅读

文章标签：

Hive Job log解析

catparseHiveJobLog.pl

use Data::Dumper;

if (!defined($ARGV[0]) or $ARGV[0] eq '')
{
    print "useage: perl parseJobHistory.pl filename\n";
    exit;
}

my $filename = $ARGV[0];
my $json = new JSON;

my %QueryStatus;
my %TaskStatus;

open FILE, "<$filename" or print("can't open $filename\n");
while(<FILE>)
{
    chomp;
    my $spacepos = index($_,' ');
    my $type = substr($_,0,$spacepos);
    my $info = substr($_,$spacepos+1,length $_);
    if(lc($type) eq 'querystart' || lc($type) eq 'queryend')
    {
        my %parseBuffer;
        $info =~ s/:" /:"/g;
        while($info =~ s/(\w+?)="(.*?)"( |$)//)
        {
            $parseBuffer{$1} = $2;
        }
        my $queryid = $parseBuffer{'QUERY_ID'};
        foreach my $key ( keys %parseBuffer )
        {
            next if $key eq 'QUERY_ID';
            $QueryStatus{$queryid}{$key} = $parseBuffer{$key};
        }
    }elsif(lc($type) eq 'taskstart' || lc($type) eq 'taskend')
    {
        my %parseBuffer;
        $info =~ s/:" /:"/g;
        while($info =~ s/(\w+?)="(.*?)"( |$)//)
        {
            $parseBuffer{$1} = $2;
        }
        my $queryid = $parseBuffer{'QUERY_ID'};
        my $taskid = $parseBuffer{'TASK_ID'};
        foreach my $key ( keys %parseBuffer )
        {
            next if $key eq 'QUERY_ID' || $key eq 'TASK_ID';
            if($key eq 'TASK_COUNTERS')
            {
                my %taskcounters;
                while ($parseBuffer{$key} =~ s/(.*?):(.*?)(,|$)//)
                {
                   $taskcounters{$1} = $2; 
                }
                foreach my $ckey ( keys %taskcounters )
                {
                    $TaskStatus{$queryid}{$taskid}{$key}{$ckey} = $taskcounters{$ckey};
                }
            }else
            {
                $TaskStatus{$queryid}{$taskid}{$key} = $parseBuffer{$key};
            }
        }
    }
    %parseBuffer = {};
}
close(FILE);

print Dumper %QueryStatus;
print Dumper %TaskStatus;