php利用SPL实现针对大文件的读取

最新推荐文章于 2022-10-17 08:02:08 发布

原创最新推荐文章于 2022-10-17 08:02:08 发布 · 972 阅读

2 ·

CC 4.0 BY-SA版权

PHP 专栏收录该内容

12 篇文章

订阅专栏

本文介绍了一种使用SPL库高效处理大型CSV文件的方法。通过实现CsvReader类，可以分段读取文件内容，避免内存溢出。该方案适用于需要逐行处理大数据集的应用场景。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

<?php

/*
php 利用SPL实现针对大文件的读取
*/
header("Content-type: text/html; charset=utf-8");
set_time_limit(0);
class CsvReader {
    private $csv_file;
    private $spl_object = null;
    private $error;
    public function __construct($csv_file = '') {
        if($csv_file && file_exists($csv_file)) {
            $this->csv_file = $csv_file;
        }
    }
    public function set_csv_file($csv_file) {
        if(!$csv_file || !file_exists($csv_file)) {
            $this->error = 'File invalid';
            return false;
        }
        $this->csv_file = $csv_file;
        $this->spl_object = null;
    }
    public function get_csv_file() {
        return $this->csv_file;
    }
    private function _file_valid($file = '') {
        $file = $file ? $file : $this->csv_file;
        if(!$file || !file_exists($file)) {
            return false;
        }
        if(!is_readable($file)) {
            return false;
        }
        return true;
    }
    private function _open_file() {
        if(!$this->_file_valid()) {
            $this->error = 'File invalid';
            return false;
        }
        if($this->spl_object == null) {
            $this->spl_object = new SplFileObject($this->csv_file, 'rb');
        }
        return true;
    }
    public function get_data($length = 0, $start = 0) {
        if(!$this->_open_file()) {
            return false;
        }
        $length = $length ? $length : $this->get_lines();
        $start = $start - 1;
        $start = ($start < 0) ? 0 : $start;
        $data = array();
        $this->spl_object->seek($start);
        while ($length-- && !$this->spl_object->eof()) {
            $data[] = $this->spl_object->fgetcsv();
            $this->spl_object->next();
        }
        return $data;
    }
    public function get_lines() {
        if(!$this->_open_file()) {
            return false;
        }
        $this->spl_object->seek(filesize($this->csv_file));
        return $this->spl_object->key();
    }
    public function get_error() {
        return $this->error;
    }
}

$csv_file = 'aaa.csv';
$csvreader = new CsvReader($csv_file);
$line_number = $csvreader->get_lines();
$page = isset($_GET['p']) ? intval($_GET['p']) : 1;
if($page == 1){
    @unlink('./result.txt');
    @unlink('./error_info_not_find.txt');
}
$num = 100;
$total_page = ceil($line_number/$num);
if($page > $total_page){
    echo "处理结束";exit();
}
$current_num = ($page-1)*$num+1;
$data = $csvreader->get_data($num,$current_num);
foreach($data as $key=>$value){
    $catId = $value[0];
    $Id = $value[1];
    $time = strtotime($value[5]);
    if($catId != "" && $Id !="" && $time !=""){
        $url = 'http://fj.china.com.cn/'.date('Y-m/d',$time).'/content_'.$Id.'.htm';
        file_put_contents('./result.txt',$url.PHP_EOL,FILE_APPEND);
    }else{
        file_put_contents('./error_info_not_find.txt',"catId:$catId Id:$Id time:$time".PHP_EOL,FILE_APPEND);
    }
}

echo "总共:$total_page 页 当前$page 页 完成度 ".round($page/$total_page*100,2)."%";
echo '<script language="javascript" type="text/javascript">window.location.href="./a.php?p='.++$page.'"</script>';
?>