hive性能测试
hive性能测试
数据准备
使用tpcds-kit工具生成数据：
sudo yum install gcc make flex bison byacc git
git clone https://github.com/gregrahn/tpcds-kit.git
cd tpcds-kit/tools
make OS=LINUX
mkdir ../data
./dsdgen -scale 50 -dir ../data/
建表语句
show databases;
create database tpcds;
create external table tpcds.dbgen_version
(
dv_version varchar(16) ,
dv_create_date date ,
dv_create_time timestamp ,
dv_cmdline_args varchar(200)
)
row format delimited
fields terminated by '|'
stored as textfile;
load data inpath '/tmp/data/dbgen_version.dat' OVERWRITE INTO TABLE tpcds.dbgen_version;
select * from tpcds.dbgen_version;
create external table tpcds.customer_address
(
ca_address_sk integer ,
ca_address_id char(16) ,
ca_street_number char(10) ,
ca_street_name varchar(60) ,
ca_street_type char(15) ,
ca_suite_number char(10) ,
ca_city varchar(60) ,
ca_county varchar(30) ,
ca_state char(2) ,
ca_zip char(10) ,
ca_country varchar(20) ,
ca_gmt_offset decimal(5,2) ,
ca_location_type char(20)
)
row format delimited
fields terminated by '|'
stored as textfile;
load data inpath '/tmp/data/customer_address.dat' overwrite into table tpcds.customer_address;
select * from tpcds.customer_address limit 100;
create external table tpcds.customer_demographics
(
cd_demo_sk integer ,
cd_gender char(1) ,
cd_marital_status char(1) ,
cd_education_status char(20) ,
cd_purchase_estimate integer ,
cd_credit_rating char(10) ,
cd_dep_count integer ,
cd_dep_employed_count integer ,
cd_dep_college_count integer
)
row format delimited
fields terminated by '|'
stored as textfile;
load data inpath '/tmp/data/customer_demographics.dat' overwrite into table tpcds.customer_demographics;
select * from tpcds.customer_demographics limit 100;
create external table tpcds.date_dim
(
d_date_sk integer ,
d_date_id char(16) ,
d_date date ,
d_month_seq integer ,
d_week_seq integer ,
d_quarter_seq integer ,
d_year integer ,
d_dow integer ,
d_moy integer ,
d_dom integer ,
d_qoy integer ,
d_fy_year integer ,
d_fy_quarter_seq integer ,
d_fy_week_seq integer ,
d_day_name char(9) ,
d_quarter_name char(6) ,
d_holiday char(1) ,
d_weekend char(1) ,
d_following_holiday char(1) ,
d_first_dom integer ,
d_last_dom integer ,
d_same_day_ly integer ,
d_same_day_lq integer ,
d_current_day char(1) ,
d_current_week char(1) ,
d_current_month char(1) ,
d_current_quarter char(1) ,
d_current_year char(1)
)
row format delimited
fields terminated by '|'
stored as textfile;
load data inpath '/tmp/data/date_dim.dat' overwrite into table tpcds.date_dim;
select * from tpcds.date_dim limit 100;
create external table tpcds.warehouse
(
w_warehouse_sk integer ,
w_warehouse_id char(16) ,
w_warehouse_name varchar(20) ,
w_warehouse_sq_ft integer ,
w_street_number char(10) ,
w_street_name varchar(60) ,
w_street_type char(15) ,
w_suite_number char(10) ,
w_city varchar(60) ,
w_county varchar(30) ,
w_state char(2) ,
w_zip char(10) ,
w_country varchar(20) ,
w_gmt_offset decimal(5,2)
)
row format delimited
fields terminated by '|'
stored as textfile;
load data inpath '/tmp/data/warehouse.dat' overwrite into table tpcds.warehouse;
select * from tpcds.warehouse limit 100;
create external table tpcds.ship_mode
(
sm_ship_mode_sk integer ,
sm_ship_mode_id char(16) ,
sm_type char(30) ,
sm_code char(10) ,
sm_carrier char(20) ,
sm_contract char(20)
)
row format delimited
fields terminated by '|'
stored as textfile;
load data inpath '/tmp/data/ship_mode.dat' overwrite into table tpcds.ship_mode;
select * from tpcds.ship_mode limit 100;
create external table tpcds.time_dim
(
t_time_sk integer ,
t_time_id char(16) ,
t_time integer ,
t_hour integer ,
t_minute integer ,
t_second integer ,
t_am_pm char(2) ,
t_shift char(20) ,
t_sub_shift char(20) ,
t_meal_time char(20)
)
row format delimited
fields terminated by '|'
stored as textfile;
load data inpath '/tmp/data/time_dim.dat' overwrite into table tpcds.time_dim;
select * from tpcds.time_dim limit 100;
create external table tpcds.reason
(
r_reason_sk integer ,
r_reason_id char(16) ,
r_reason_desc char(100)
)
row format delimited
fields terminated by '|'
stored as textfile;
load data inpath '/tmp/data/reason.dat' overwrite into table tpcds.reason;
select * from tpcds.reason limit 100;
create external table tpcds.income_band
(
ib_income_band_sk integer ,
ib_lower_bound integer ,
ib_upper_bound integer
)
row format delimited
fields terminated by '|'
stored as textfile;
load data inpath '/tmp/data/income_band.dat' overwrite into table tpcds.income_band;
select * from tpcds.income_band limit 100;
create external table tpcds.item
(
i_item_sk integer ,
i_item_id char(16) ,
i_rec_start_date date ,
i_rec_end_date date ,
i_item_desc varchar(200) ,
i_current_price decimal(7,2) ,
i_wholesale_cost decimal(7,2) ,
i_brand_id integer ,
i_brand char(50) ,
i_class_id integer ,
i_class char(50) ,
i_category_id integer ,
i_category char(50) ,
i_manufact_id integer ,
i_manufact char(50) ,
i_size char(20) ,
i_formulation char(20) ,
i_color char(20) ,
i_units char(10) ,
i_container char(10) ,
i_manager_id integer ,
i_product_name char(50)
)
row format delimited
fields terminated by '|'
stored as textfile;
load data inpath '/tmp/data/item.dat' overwrite into table tpcds.item;
select * from tpcds.item limit 100;
create external table tpcds.store
(
s_store_sk integer ,
s_store_id char(16) ,
s_rec_start_date date ,
s_rec_end_date date ,
s_closed_date_sk integer ,
s_store_name varchar(50) ,
s_number_employees integer ,
s_floor_space integer ,
s_hours char(20) ,
s_manager varchar(40) ,
s_market_id integer ,
s_geography_class varchar(100) ,
s_market_desc varchar(100) ,
s_market_manager varchar(40) ,
s_division_id integer ,
s_division_name varchar(50) ,
s_company_id integer ,
s_company_name varchar(50) ,
s_street_number varchar(10) ,
s_street_name varchar(60) ,
s_street_type char(15) ,
s_suite_number char(10) ,
s_city varchar(60) ,
s_county varchar(30) ,
s_state char(2) ,
s_zip char(10) ,
s_country varchar(20) ,
s_gmt_offset decimal(5,2) ,
s_tax_precentage decimal(5,2)
)
row format delimited
fields terminated by '|'
stored as textfile;
load data inpath '/tmp/data/store.dat' overwrite into table tpcds.store;
select * from tpcds.store limit 100;
create external table tpcds.call_center
(
cc_call_center_sk integer ,
cc_call_center_id char(16) ,
cc_rec_start_date date ,
cc_rec_end_date date ,
cc_closed_date_sk integer ,
cc_open_date_sk integer ,
cc_name varchar(50) ,
cc_class varchar(50) ,
cc_employees integer ,
cc_sq_ft integer ,
cc_hours char(20) ,
cc_manager varchar(40) ,
cc_mkt_id integer ,
cc_mkt_class char(50) ,
cc_mkt_desc varchar(100) ,
cc_market_manager varchar(40) ,
cc_division integer ,
cc_division_name varchar(50) ,
cc_company integer ,
cc_company_name char(50) ,
cc_street_number char(10) ,
cc_street_name varchar(60) ,
cc_street_type char(15) ,
cc_suite_number char(10) ,
cc_city varchar(60) ,
cc_county varchar(30) ,
cc_state char(2) ,
cc_zip char(10) ,
cc_country varchar(20) ,
cc_gmt_offset decimal(5,2) ,
cc_tax_percentage decimal(5,2)
)
row format delimited
fields terminated by '|'
stored as textfile;
load data inpath '/tmp/data/call_center.dat' overwrite into table tpcds.call_center;
select * from tpcds.call_center limit 100;
create external table tpcds.customer
(
c_customer_sk integer ,
c_customer_id char(16) ,
c_current_cdemo_sk integer ,
c_current_hdemo_sk integer ,
c_current_addr_sk integer ,
c_first_shipto_date_sk integer ,
c_first_sales_date_sk integer ,
c_salutation char(10) ,
c_first_name char(20) ,
c_last_name char(30) ,
c_preferred_cust_flag char(1) ,
c_birth_day integer ,
c_birth_month integer ,
c_birth_year integer ,
c_birth_country varchar(20) ,
c_login char(13) ,
c_email_address char(50) ,
c_last_review_date char(10)
)
row format delimited
fields terminated by '|'
stored as textfile;
load data inpath '/tmp/data/customer.dat' overwrite into table tpcds.customer;
select * from tpcds.customer limit 100;
create external table tpcds.web_site
(
web_site_sk integer ,
web_site_id char(16) ,
web_rec_start_date date ,
web_rec_end_date date ,
web_name varchar(50) ,
web_open_date_sk integer ,
web_close_date_sk integer ,
web_class varchar(50) ,
web_manager varchar(40) ,
web_mkt_id integer ,
web_mkt_class varchar(50) ,
web_mkt_desc varchar(100) ,
web_market_manager varchar(40) ,
web_company_id integer ,
web_company_name char(50) ,
web_street_number char(10) ,
web_street_name varchar(60) ,
web_street_type char(15) ,
web_suite_number char(10) ,
web_city varchar(60) ,
web_county varchar(30) ,
web_state char(2) ,
web_zip char(10) ,
web_country varchar(20) ,
web_gmt_offset decimal(5,2) ,
web_tax_percentage decimal(5,2)
)
row format delimited
fields terminated by '|'
stored as textfile;
load data inpath '/tmp/data/web_site.dat' overwrite into table tpcds.web_site;
select * from tpcds.web_site limit 100;
create external table tpcds.store_returns
(
sr_returned_date_sk integer ,
sr_return_time_sk integer ,
sr_item_sk integer ,
sr_customer_sk integer ,
sr_cdemo_sk integer ,
sr_hdemo_sk integer ,
sr_addr_sk integer ,
sr_store_sk integer ,
sr_reason_sk integer ,
sr_ticket_number integer ,
sr_return_quantity integer ,
sr_return_amt decimal(7,2) ,
sr_return_tax decimal(7,2) ,
sr_return_amt_inc_tax decimal(7,2) ,
sr_fee decimal(7,2) ,
sr_return_ship_cost decimal(7,2) ,
sr_refunded_cash decimal(7,2) ,
sr_reversed_charge decimal(7,2) ,
sr_store_credit decimal(7,2) ,
sr_net_loss decimal(7,2)
)
row format delimited
fields terminated by '|'
stored as textfile;
load data inpath '/tmp/data/store_returns.dat' overwrite into table tpcds.store_returns;
select * from tpcds.store_returns limit 100;
create external table tpcds.household_demographics
(
hd_demo_sk integer ,
hd_income_band_sk integer ,
hd_buy_potential char(15) ,
hd_dep_count integer ,
hd_vehicle_count integer
)
row format delimited
fields terminated by '|'
stored as textfile;
load data inpath '/tmp/data/household_demographics.dat' overwrite into table tpcds.household_demographics;
select * from tpcds.household_demographics limit 100;
create external table tpcds.web_page
(
wp_web_page_sk integer ,
wp_web_page_id char(16) ,
wp_rec_start_date date ,
wp_rec_end_date date ,
wp_creation_date_sk integer ,
wp_access_date_sk integer ,
wp_autogen_flag char(1) ,
wp_customer_sk integer ,
wp_url varchar(100) ,
wp_type char(50) ,
wp_char_count integer ,
wp_link_count integer ,
wp_image_count integer ,
wp_max_ad_count integer
)
row format delimited
fields terminated by '|'
stored as textfile;
load data inpath '/tmp/data/web_page.dat' overwrite into table tpcds.web_page;
select * from tpcds.web_page limit 100;
create external table tpcds.promotion
(
p_promo_sk integer ,
p_promo_id char(16) ,
p_start_date_sk integer ,
p_end_date_sk integer ,
p_item_sk integer ,
p_cost decimal(15,2) ,
p_response_target integer ,
p_promo_name char(50) ,
p_channel_dmail char(1) ,
p_channel_email char(1) ,
p_channel_catalog char(1) ,
p_channel_tv char(1) ,
p_channel_radio char(1) ,
p_channel_press char(1) ,
p_channel_event char(1) ,
p_channel_demo char(1) ,
p_channel_details varchar(100) ,
p_purpose char(15) ,
p_discount_active char(1)
)
row format delimited
fields terminated by '|'
stored as textfile;
load data inpath '/tmp/data/promotion.dat' overwrite into table tpcds.promotion;
select * from tpcds.promotion limit 100;
create external table tpcds.catalog_page
(
cp_catalog_page_sk integer ,
cp_catalog_page_id char(16) ,
cp_promo_id integer ,
cp_start_date_sk integer ,
cp_end_date_sk integer ,
cp_department varchar(50) ,
cp_catalog_number integer ,
cp_catalog_page_number integer ,
cp_description varchar(100) ,
cp_type varchar(100)
)
row format delimited
fields terminated by '|'
stored as textfile;
load data inpath '/tmp/data/catalog_page.dat' overwrite into table tpcds.catalog_page;
select * from tpcds.catalog_page limit 100;
create external table tpcds.inventory
(
inv_date_sk integer ,
inv_item_sk integer ,
inv_warehouse_sk integer ,
inv_quantity_on_hand integer
)
row format delimited
fields terminated by '|'
stored as textfile;
load data inpath '/tmp/data/inventory.dat' overwrite into table tpcds.inventory;
select * from tpcds.inventory limit 100;
create external table tpcds.catalog_returns
(
cr_returned_date_sk integer ,
cr_returned_time_sk integer ,
cr_item_sk integer ,
cr_ship_date_sk integer ,
cr_refunded_customer_sk integer ,
cr_refunded_cdemo_sk integer ,
cr_refunded_hdemo_sk integer ,
cr_refunded_addr_sk integer ,
cr_returning_customer_sk integer ,
cr_returning_cdemo_sk integer ,
cr_returning_hdemo_sk integer ,
cr_returning_addr_sk integer ,
cr_call_center_sk integer ,
cr_catalog_page_sk integer ,
cr_ship_mode_sk integer ,
cr_warehouse_sk integer ,
cr_reason_sk integer ,
cr_order_number integer ,
cr_return_quantity integer ,
cr_return_amount decimal(7,2) ,
cr_return_tax decimal(7,2) ,
cr_return_amt_inc_tax decimal(7,2) ,
cr_fee decimal(7,2) ,
cr_return_ship_cost decimal(7,2) ,
cr_refunded_cash decimal(7,2) ,
cr_reversed_charge decimal(7,2) ,
cr_store_credit decimal(7,2) ,
cr_net_loss decimal(7,2)
)
row format delimited
fields terminated by '|'
stored as textfile;
load data inpath '/tmp/data/catalog_returns.dat' overwrite into table tpcds.catalog_returns;
select * from tpcds.catalog_returns limit 100;
create external table tpcds.web_returns
(
wr_returned_date_sk integer ,
wr_returned_time_sk integer ,
wr_item_sk integer ,
wr_refunded_customer_sk integer ,
wr_refunded_cdemo_sk integer ,
wr_refunded_hdemo_sk integer ,
wr_refunded_addr_sk integer ,
wr_returning_customer_sk integer ,
wr_returning_cdemo_sk integer ,
wr_returning_hdemo_sk integer ,
wr_returning_addr_sk integer ,
wr_web_page_sk integer ,
wr_reason_sk integer ,
wr_order_number integer ,
wr_return_quantity integer ,
wr_return_amt decimal(7,2) ,
wr_return_tax decimal(7,2) ,
wr_return_amt_inc_tax decimal(7,2) ,
wr_fee decimal(7,2) ,
wr_return_ship_cost decimal(7,2) ,
wr_refunded_cash decimal(7,2) ,
wr_reversed_charge decimal(7,2) ,
wr_account_credit decimal(7,2) ,
wr_net_loss decimal(7,2)
)
row format delimited
fields terminated by '|'
stored as textfile;
load data inpath '/tmp/data/web_returns.dat' overwrite into table tpcds.web_returns;
select * from tpcds.web_returns limit 100;
create external table tpcds.web_sales
(
ws_sold_date_sk integer ,
ws_sold_time_sk integer ,
ws_ship_date_sk integer ,
ws_item_sk integer ,
ws_bill_customer_sk integer ,
ws_bill_cdemo_sk integer ,
ws_bill_hdemo_sk integer ,
ws_bill_addr_sk integer ,
ws_ship_customer_sk integer ,
ws_ship_cdemo_sk integer ,
ws_ship_hdemo_sk integer ,
ws_ship_addr_sk integer ,
ws_web_page_sk integer ,
ws_web_site_sk integer ,
ws_ship_mode_sk integer ,
ws_warehouse_sk integer ,
ws_promo_sk integer ,
ws_order_number integer ,
ws_quantity integer ,
ws_wholesale_cost decimal(7,2) ,
ws_list_price decimal(7,2) ,
ws_sales_price decimal(7,2) ,
ws_ext_discount_amt decimal(7,2) ,
ws_ext_sales_price decimal(7,2) ,
ws_ext_wholesale_cost decimal(7,2) ,
ws_ext_list_price decimal(7,2) ,
ws_ext_tax decimal(7,2) ,
ws_coupon_amt decimal(7,2) ,
ws_ext_ship_cost decimal(7,2) ,
ws_net_paid decimal(7,2) ,
ws_net_paid_inc_tax decimal(7,2) ,
ws_net_paid_inc_ship decimal(7,2) ,
ws_net_paid_inc_ship_tax decimal(7,2) ,
ws_net_profit decimal(7,2)
)
row format delimited
fields terminated by '|'
stored as textfile;
load data inpath '/tmp/data/web_sales.dat' overwrite into table tpcds.web_sales;
select * from tpcds.web_sales limit 100;
create external table tpcds.catalog_sales
(
cs_sold_date_sk integer ,
cs_sold_time_sk integer ,
cs_ship_date_sk integer ,
cs_bill_customer_sk integer ,
cs_bill_cdemo_sk integer ,
cs_bill_hdemo_sk integer ,
cs_bill_addr_sk integer ,
cs_ship_customer_sk integer ,
cs_ship_cdemo_sk integer ,
cs_ship_hdemo_sk integer ,
cs_ship_addr_sk integer ,
cs_call_center_sk integer ,
cs_catalog_page_sk integer ,
cs_ship_mode_sk integer ,
cs_warehouse_sk integer ,
cs_item_sk integer ,
cs_promo_sk integer ,
cs_order_number integer ,
cs_quantity integer ,
cs_wholesale_cost decimal(7,2) ,
cs_list_price decimal(7,2) ,
cs_sales_price decimal(7,2) ,
cs_ext_discount_amt decimal(7,2) ,
cs_ext_sales_price decimal(7,2) ,
cs_ext_wholesale_cost decimal(7,2) ,
cs_ext_list_price decimal(7,2) ,
cs_ext_tax decimal(7,2) ,
cs_coupon_amt decimal(7,2) ,
cs_ext_ship_cost decimal(7,2) ,
cs_net_paid decimal(7,2) ,
cs_net_paid_inc_tax decimal(7,2) ,
cs_net_paid_inc_ship decimal(7,2) ,
cs_net_paid_inc_ship_tax decimal(7,2) ,
cs_net_profit decimal(7,2)
)
row format delimited
fields terminated by '|'
stored as textfile;
load data inpath '/tmp/data/catalog_sales.dat' overwrite into table tpcds.catalog_sales;
select * from tpcds.catalog_sales limit 100;
create external table tpcds.store_sales
(
ss_sold_date_sk integer ,
ss_sold_time_sk integer ,
ss_item_sk integer ,
ss_customer_sk integer ,
ss_cdemo_sk integer ,
ss_hdemo_sk integer ,
ss_addr_sk integer ,
ss_store_sk integer ,
ss_promo_sk integer ,
ss_ticket_number integer ,
ss_quantity integer ,
ss_wholesale_cost decimal(7,2) ,
ss_list_price decimal(7,2) ,
ss_sales_price decimal(7,2) ,
ss_ext_discount_amt decimal(7,2) ,
ss_ext_sales_price decimal(7,2) ,
ss_ext_wholesale_cost decimal(7,2) ,
ss_ext_list_price decimal(7,2) ,
ss_ext_tax decimal(7,2) ,
ss_coupon_amt decimal(7,2) ,
ss_net_paid decimal(7,2) ,
ss_net_paid_inc_tax decimal(7,2) ,
ss_net_profit decimal(7,2)
)
row format delimited
fields terminated by '|'
stored as textfile;
load data inpath '/tmp/data/store_sales.dat' overwrite into table tpcds.store_sales;
select * from tpcds.store_sales limit 100;
Powered by Waline v2.14.1