大数据应用1：Spark分析处理Oracle大表

0、制造测试数据

https://blog.youkuaiyun.com/lizhangyong1989/article/details/45013509

 

生产一千万条测试数据总共四个字段，每次插入一百万的数据耗时110秒左右

create table TestTable as

select rownum as id,

to_char(sysdate + rownum / 24 / 3600, 'yyyy-mm-dd hh24:mi:ss') as inc_datetime,

trunc(dbms_random.value(0, 100)) as random_id,

dbms_random.string('x', 20) random_string

from dual

connect by level <= 1000000;

因为不能一次性插入一千万，所以分十次，一千万数据量插入约为1200秒即20分钟

insert into TestTable

(ID, INC_DATETIME, RANDOM_ID, RANDOM_STRING)

select rownum as id,

to_char(sysdate + rownum / 24 / 3600, 'yyyy-mm-dd hh24:mi:ss') as inc_datetime,

trunc(dbms_random.value(0, 100)) as random_id,

dbms_random.string('x', 20) random_string

from dual

connect by level <= 1000000;

 

1、在Oracl