Logging function is shell script not capturing all the lines -
i have written shell script logging , logging function captures first 2 lines. not capturing whole log. please validate script , let me know going wrong.
#!/bin/bash #### function logging #### dir=$(pwd) log_file_name="sqoop.log" log_time=$(date "+%y-%m-%d %h:%m:%s") write_to_log(){ while read -r text if [ ! -f "$dir/$log_file_name" ]; touch "$dir/$log_file_name" else echo "$log_time" : "$text" | tee -a "$dir/$log_file_name"; fi done } echo "importing xxccs_ds_sahdr_core table" ssh $1 "sqoop import -d mapred.child.java.opts='\-djava.security.egd=file:/dev/../dev/urandom' --connect '$2' --username $3 --password $4 --query \"select contract_id,contract_number,contract_sts_code,contract_status,service_line_id,service_line_name,service_line_sts_code,service_line_status,service_line_number,hdr_cust_account_id,hdr_cust_account_number,hdr_party_id,hdr_customer_name,line_cust_account_id,line_cust_account_number,line_party_id,line_customer_name,bill_to_site_use_id,bill_to_site_use_name,bill_to_site_use_sts,bill_to_cust_acct_id,bill_to_cust_acct_number,bill_to_cust_acct_sts,bill_to_cust_acct_site_id,bill_to_cust_acct_site_sts,bill_to_party_id,bill_to_party_number,bill_to_customer_name,bill_to_party_sts,bill_to_party_site_id,bill_to_party_site_sts,bill_to_location_id,bill_to_country,bill_to_state_prov,bill_to_city,bill_to_postal_code,coverage_template_id,coverage_template_name,coverage_template_desc,coverage_begin_date,coverage_end_date,service_sales_rep_usr_id,service_sales_rep_usr_name,service_line_inventory_item_id,billto_csc_id,billto_csc_name,billto_gu_id,billto_gu_name,bill_to_customer_sales_channel,currency_code,contract_list_amt,contract_net_amt,usd_contract_list_amt,usd_contract_net_amt,account_team,renewal_ssr,renewal_team,sales_region,advance_contract_flag,earliest_end_date,service_line_terminated_date,svc_line_termination_reason,contract_terminated_date,usd_conv_rate,price_type,net_value,coterm_date,marketing_program,master_agreement_end_date,discount_percent,prorate_flag,rgp_id,creation_date,created_by,last_update_date,last_updated_by,last_updated_login,request_id,program_update_date,program_application_id,program_id,attribute_category,attribute1,attribute2,attribute3,attribute4,attribute5,attribute6,attribute7,attribute8,attribute9,attribute10,attribute11,attribute12,attribute13,attribute14,attribute15,attribute16,attribute17,attribute18,attribute19,attribute20,account_ssr,shell_contract_flag,contract_start_date,contract_end_date xxccs_ds_sahdr_core \\\$conditions \" --split-by contract_number -m 4 --null-string '\\\\n' --null-non-string '\\\\n' --hive-delims-replacement '<eol>' --boundary-query 'select (select min(contract_number) xxccs_ds_sahdr_core) minid ,(select max(contract_number) xxccs_ds_sahdr_core) maxid dual' --target-dir $5/xxccs_ds_sahdr_core --hive-import --hive-overwrite --hive-table $6.xxccs_ds_sahdr_core --map-column-hive contract_id=bigint,service_line_id=bigint,hdr_cust_account_id=bigint,hdr_party_id=bigint,line_cust_account_id=bigint,line_party_id=bigint,bill_to_site_use_id=bigint,bill_to_cust_acct_id=bigint,bill_to_cust_acct_site_id=bigint,bill_to_party_id=bigint,bill_to_party_site_id=bigint,bill_to_location_id=bigint,coverage_template_id=bigint,coverage_begin_date=timestamp,coverage_end_date=timestamp,service_line_inventory_item_id=bigint,billto_csc_id=bigint,billto_gu_id=bigint,contract_list_amt=bigint,contract_net_amt=bigint,usd_contract_list_amt=bigint,usd_contract_net_amt=bigint,earliest_end_date=timestamp,service_line_terminated_date=timestamp,contract_terminated_date=timestamp,usd_conv_rate=bigint,net_value=bigint,master_agreement_end_date=timestamp,discount_percent=bigint,prorate_flag=string,rgp_id=bigint,creation_date=timestamp,created_by=bigint,last_update_date=timestamp,last_updated_by=bigint,last_updated_login=bigint,request_id=bigint,program_update_date=timestamp,program_application_id=bigint,program_id=bigint,attribute_category=string,attribute1=string,attribute2=string,attribute3=string,attribute4=string,attribute5=string,attribute6=string,attribute7=string,attribute8=string,attribute9=string,attribute10=string,attribute11=string,attribute12=string,attribute13=string,attribute14=string,attribute15=string,attribute16=string,attribute17=string,attribute18=string,attribute19=string,attribute20=string,account_ssr=string,shell_contract_flag=string,contract_start_date=timestamp,contract_end_date=timestamp" | write_to_log ssh $1 "sqoop import -d mapred.child.java.opts='\-djava.security.egd=file:/dev/../dev/urandom' --connect '$2' --username $3 --password $4 --query \"select contract_id,contract_number,contract_sts_code,contract_status,service_line_id,service_line_name,service_line_sts_code,service_line_status,service_line_number,hdr_cust_account_id,hdr_cust_account_number,hdr_party_id,hdr_customer_name,line_cust_account_id,line_cust_account_number,line_party_id,line_customer_name,bill_to_site_use_id,bill_to_site_use_name,bill_to_site_use_sts,bill_to_cust_acct_id,bill_to_cust_acct_number,bill_to_cust_acct_sts,bill_to_cust_acct_site_id,bill_to_cust_acct_site_sts,bill_to_party_id,bill_to_party_number,bill_to_customer_name,bill_to_party_sts,bill_to_party_site_id,bill_to_party_site_sts,bill_to_location_id,bill_to_country,bill_to_state_prov,bill_to_city,bill_to_postal_code,coverage_template_id,coverage_template_name,coverage_template_desc,coverage_begin_date,coverage_end_date,service_sales_rep_usr_id,service_sales_rep_usr_name,service_line_inventory_item_id,billto_csc_id,billto_csc_name,billto_gu_id,billto_gu_name,bill_to_customer_sales_channel,currency_code,contract_list_amt,contract_net_amt,usd_contract_list_amt,usd_contract_net_amt,account_team,renewal_ssr,renewal_team,sales_region,advance_contract_flag,earliest_end_date,service_line_terminated_date,svc_line_termination_reason,contract_terminated_date,usd_conv_rate,price_type,net_value,coterm_date,marketing_program,master_agreement_end_date,discount_percent,prorate_flag,rgp_id,creation_date,created_by,last_update_date,last_updated_by,last_updated_login,request_id,program_update_date,program_application_id,program_id,attribute_category,attribute1,attribute2,attribute3,attribute4,attribute5,attribute6,attribute7,attribute8,attribute9,attribute10,attribute11,attribute12,attribute13,attribute14,attribute15,attribute16,attribute17,attribute18,attribute19,attribute20,account_ssr,shell_contract_flag,contract_start_date,contract_end_date xxccs_ds_sahdr_core \\\$conditions \" --split-by contract_number -m 4 --null-string '\\\\n' --null-non-string '\\\\n' --hive-delims-replacement '<eol>' --boundary-query 'select (select min(contract_number) xxccs_ds_sahdr_core) minid ,(select max(contract_number) xxccs_ds_sahdr_core) maxid dual' --target-dir $5/xxccs_ds_sahdr_core --hive-import --hive-overwrite --hive-table $6.xxccs_ds_sahdr_core --map-column-hive contract_id=bigint,service_line_id=bigint,hdr_cust_account_id=bigint,hdr_party_id=bigint,line_cust_account_id=bigint,line_party_id=bigint,bill_to_site_use_id=bigint,bill_to_cust_acct_id=bigint,bill_to_cust_acct_site_id=bigint,bill_to_party_id=bigint,bill_to_party_site_id=bigint,bill_to_location_id=bigint,coverage_template_id=bigint,coverage_begin_date=timestamp,coverage_end_date=timestamp,service_line_inventory_item_id=bigint,billto_csc_id=bigint,billto_gu_id=bigint,contract_list_amt=bigint,contract_net_amt=bigint,usd_contract_list_amt=bigint,usd_contract_net_amt=bigint,earliest_end_date=timestamp,service_line_terminated_date=timestamp,contract_terminated_date=timestamp,usd_conv_rate=bigint,net_value=bigint,master_agreement_end_date=timestamp,discount_percent=bigint,prorate_flag=string,rgp_id=bigint,creation_date=timestamp,created_by=bigint,last_update_date=timestamp,last_updated_by=bigint,last_updated_login=bigint,request_id=bigint,program_update_date=timestamp,program_application_id=bigint,program_id=bigint,attribute_category=string,attribute1=string,attribute2=string,attribute3=string,attribute4=string,attribute5=string,attribute6=string,attribute7=string,attribute8=string,attribute9=string,attribute10=string,attribute11=string,attribute12=string,attribute13=string,attribute14=string,attribute15=string,attribute16=string,attribute17=string,attribute18=string,attribute19=string,attribute20=string,account_ssr=string,shell_contract_flag=string,contract_start_date=timestamp,contract_end_date=timestamp" | write_to_log
Comments
Post a Comment