4.2抽取web数据

原创已于 2022-11-26 18:57:10 修改 · 197 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#etl

于 2022-11-26 17:37:54 首次发布

4.2.1HTML网页的数据抽取
创建转换
在这里插入图片描述
配置自定义常量数据控件，在元数据下加入filename

配置自定义常量数据控件，在数据下写入网址
配置HTTP client控件

java代码：

import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.mysql.jdbc.Connection;
import com.mysql.jdbc.PreparedStatement;
import java.io.InputStream;
import java.io.RandomAccessFile;
import java.net.URL;
import java.net.URLConnection;
 
private String result;
private String contents;
private Connection connection = null;
 
public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
  if (first) {
    first = false;
 
    /* TODO: Your code here. (Using info fields)
    FieldHelper infoField = get(Fields.Info, "info_field_name");
    RowSet infoStream = findInfoRowSet("info_stream_tag");
    Object[] infoRow = null;
    int infoRowCount = 0;
    // Read all rows from info step before calling getRow() method, which returns first row from any
    // input rowset. As rowMeta for info and input steps varies getRow() can lead to errors.
    while((infoRow = getRowFrom(infoStream)) != null){
      // do something with info data
      infoRowCount++;
    }
    */
  }
	try{
 
		URL url = new URL("https://movie.douban.com/");
 
		URLConnection conn = url.openConnection();
 
		conn.setRequestProperty("accept","*/*");
 
		conn.setRequestProperty("connection","Keep-Alive");
 
		conn.setRequestProperty("user-agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36");
		conn.connect();
 
		InputStream input = conn.getInputStream();
		byte[] buffer = new byte[1024];
		int hasRead;
		int length = 0;
 
		String msg = "";
 
        //输出到一个txt文件中
        //FileWriter fw = new FileWriter("D:\\豆瓣电影排行榜.txt");
		RandomAccessFile raf = new RandomAccessFile("D:\\output\\豆瓣电影排行榜.txt","rw");
 
		while((hasRead =input.read(buffer)) != -1)
		{
			raf.write(buffer);
			length += hasRead;
			System.out.println("爬取进度："+length);
		}
		raf.close();
		logDebug("爬取完成！");
 
 
	} catch (Exception e) {
		logDebug("异常");
		e.printStackTrace();
	}
 
 
  Object[] r = getRow();
 
  if (r == null) {
    setOutputDone();
    return false;
  }
 
  // It is always safest to call createOutputRow() to ensure that your output row's Object[] is large
  // enough to handle any new fields you are creating in this step.
  r = createOutputRow(r, data.outputRowMeta.size());
 
  /* TODO: Your code here. (See Sample)
  // Get the value from an input field
  String foobar = get(Fields.In, "a_fieldname").getString(r);
  foobar += "bar";
    
  // Set a value in a new output field
  get(Fields.Out, "output_fieldname").setValue(r, foobar);
  */
  // Send the row on to the next step.
  putRow(data.outputRowMeta, r);
 
  return true;
}