From 39e356e2ffc06d0e221819e8386294c8c4dcac9b Mon Sep 17 00:00:00 2001 From: Daeng Deni Mardaeni Date: Wed, 21 May 2025 08:17:47 +0700 Subject: [PATCH] refactor(webstatement): update data processing and migration logic - Menonaktifkan pemanggilan fungsi-fungsi yang tidak diperlukan untuk data processing di `MigrasiController`: - `processTransactionData`, `processStmtNarrFormatData`, `processAccountData`, dan `ProcessCompanyData`. - Mengubah argumen di beberapa pemanggilan fungsi untuk data processing agar lebih spesifik ke periode tertentu: - Mengganti `$periods` menjadi `['20250512']` untuk `processArrangementData`, `processBillDetailData`, `processFundsTransferData`, dan `processStmtEntryData`. - Memodifikasi logika skipping folder `_parameter` pada `ProcessFtTxnTypeConditionJob` dengan mengomentari proses pengecekan. - Mengubah delimiter `fgetcsv` pada `ProcessBillDetailDataJob` dari `;` menjadi `~`. - Menambahkan file migrasi baru untuk mengubah tipe kolom pada tabel `temp_fund_transfer`: - Mengubah semua kolom pada tabel menjadi tipe `text` untuk mendukung data yang lebih besar. - Menyediakan metode `down` untuk rollback tipe data kembali ke `string`. Perubahan ini bertujuan untuk meningkatkan efisiensi proses data, mendukung fleksibilitas data lebih besar pada migrasi database, serta membuka jalan untuk refaktor atau penghapusan fungsi yang tidak digunakan. Signed-off-by: Daeng Deni Mardaeni --- app/Http/Controllers/MigrasiController.php | 16 +++---- app/Jobs/ProcessBillDetailDataJob.php | 2 +- app/Jobs/ProcessFtTxnTypeConditionJob.php | 4 +- ...ange_temp_fund_transfer_fields_to_text.php | 44 +++++++++++++++++++ 4 files changed, 55 insertions(+), 11 deletions(-) create mode 100644 database/migrations/2025_05_21_010009_change_temp_fund_transfer_fields_to_text.php diff --git a/app/Http/Controllers/MigrasiController.php b/app/Http/Controllers/MigrasiController.php index c7dba57..9cdc0ae 100644 --- a/app/Http/Controllers/MigrasiController.php +++ b/app/Http/Controllers/MigrasiController.php @@ -151,9 +151,9 @@ class MigrasiController extends Controller // Get all directories (periods) in the SFTP disk $allDirectories = $disk->directories(); - $this->processTransactionData(['_parameter']); + //$this->processTransactionData(['_parameter']); $this->processStmtNarrParamData(['_parameter']); - $this->processStmtNarrFormatData(['_parameter']); + //$this->processStmtNarrFormatData(['_parameter']); $this->ProcessFtTxnTypeConditioData(['_parameter']); @@ -172,13 +172,13 @@ class MigrasiController extends Controller } $this->processCustomerData(['20250519']); - $this->processAccountData(['20250519']); - $this->processArrangementData($periods); - $this->processBillDetailData($periods); - $this->processFundsTransferData($periods); - $this->processStmtEntryData($periods); + //$this->processAccountData(['20250519']); + $this->processArrangementData(['20250512']); + $this->processBillDetailData(['20250512']); + $this->processFundsTransferData(['20250512']); + $this->processStmtEntryData(['20250512']); - $this->ProcessCompanyData($periods); + //$this->ProcessCompanyData($periods); $this->ProcessDataCaptureData($periods); return response()->json(['message' => 'Data processing job has been successfully']); diff --git a/app/Jobs/ProcessBillDetailDataJob.php b/app/Jobs/ProcessBillDetailDataJob.php index 2e5927a..38b48a5 100644 --- a/app/Jobs/ProcessBillDetailDataJob.php +++ b/app/Jobs/ProcessBillDetailDataJob.php @@ -70,7 +70,7 @@ class ProcessBillDetailDataJob implements ShouldQueue $headers = (new TempBillDetail())->getFillable(); $rowCount = 0; - while (($row = fgetcsv($handle, 0, ";")) !== false) { + while (($row = fgetcsv($handle, 0, "~")) !== false) { $rowCount++; if (count($headers) === count($row)) { diff --git a/app/Jobs/ProcessFtTxnTypeConditionJob.php b/app/Jobs/ProcessFtTxnTypeConditionJob.php index 5988047..0c693fb 100644 --- a/app/Jobs/ProcessFtTxnTypeConditionJob.php +++ b/app/Jobs/ProcessFtTxnTypeConditionJob.php @@ -46,10 +46,10 @@ class ProcessFtTxnTypeConditionJob implements ShouldQueue foreach ($this->periods as $period) { // Skip the _parameter folder - if ($period === '_parameter') { + /*if ($period === '_parameter') { Log::info("Skipping _parameter folder"); continue; - } + }*/ // Construct the filepath based on the period folder name $filePath = "$period/{$this->filename}"; diff --git a/database/migrations/2025_05_21_010009_change_temp_fund_transfer_fields_to_text.php b/database/migrations/2025_05_21_010009_change_temp_fund_transfer_fields_to_text.php new file mode 100644 index 0000000..8179bea --- /dev/null +++ b/database/migrations/2025_05_21_010009_change_temp_fund_transfer_fields_to_text.php @@ -0,0 +1,44 @@ +text($field)->nullable()->change(); + } + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + $fields = '_id,ref_no,transaction_type,debit_acct_no,reserved_debit_acct_name,reserved_debit_cust_address,reserved_debit_cust_street,reserved_debt_cus_town_con,reserved_debit_cus_post_cd,reserved_debt_cus_country,reserved_debit_cus_name_1,reserved_debit_cus_name_2,reserved_debit_cus_legal_id,currency_mkt_dr,debit_currency,debit_amount,debit_value_date,debit_their_ref,credit_their_ref,credit_acct_no,reserved_credit_acct_name,reserved_cred_cus_address,reserved_cred_cus_street,reserved_cred_cus_town_con,reserved_cred_cus_pos_code,reserved_cred_cus_country,reserved_cred_cus_name_1,reserved_cred_cus_name_2,reserved_cred_cus_legal_id,currency_mkt_cr,credit_currency,credit_amount,credit_value_date,treasury_rate,neg_dealer_refno,processing_date,ordering_cust,in_ordering_cus,ordering_bank,in_ordering_bk,acct_with_bank,reserved_act_with_bank_name,acct_with_bk,ben_acct_no,reserved_ben_acct_name,in_ben_acct_no,ben_customer,in_ben_customer,ben_bank,reserved_ben_bank_name,in_ben_bank,cheque_number,payment_details,in_pay_details,bc_bank_sort_code,receiver_bank,reserved_rcvr_bank_name,rec_corr_bank,reserved_rec_corr_bank_name,intermed_bank,reserved_intermed_bank_name,mailing,pay_method,ben_our_charges,charges_acct_no,charge_com_display,commission_code,commission_type,commission_amt,charge_code,charge_type,charge_amt,customer_spread,base_currency,profit_centre_cust,profit_centre_dept,return_to_dept,priority_txn,bk_to_bk_info,exposure_date,fed_funds,position_type,no_of_batch_items,free_text_msgto,message,local_ref,tax_type,tax_amt,amount_debited,amount_credited,total_charge_amt,total_tax_amount,customer_rate,in_rec_corr_bk,inward_pay_type,in_send_corr_bk,telex_from_cust,delivery_inref,delivery_outref,credit_comp_code,debit_comp_code,status,delivery_mkr,acct_with_bk_acno,loc_amt_debited,loc_amt_credited,loc_tot_tax_amt,local_charge_amt,loc_pos_chgs_amt,mktg_exch_profit,rate_input_mkr,cust_group_level,debit_customer,credit_customer,send_payment_y_n,dr_advice_reqd_y_n,cr_advice_reqd_y_n,deal_market,stmt_nos,override,record_status,curr_no,inputter,date_time,authoriser,co_code,reserved_cont_comp_mne,dept_code,auditor_code,audit_date_time,charged_customer,in_reason_ove,dealer_desk,reserved_msg_reference,recalc_fwd_rate,drawn_account,accounting_date,instrctn_code,commission_for,charge_for,expected_recs_id,tot_rec_comm,tot_rec_comm_lcl,tot_rec_chg,tot_rec_chg_lcl,in_bk_to_bk,in_bk_to_bk_info,coll_rem_bk,reserved_coll_rem_bk_name,cheq_type,reserved_1,related_msg,time_ind,in_time_ind1,send_to_party,bk_to_bk_out,message_type,reversal_mkr,related_ref,in_instr_code,in_process_err,in_swift_msg,acct_with_bank_acc,reserved_ac_with_bank_name,in_acct_bank_acc,rec_corr_bank_acc,reserved_rec_corr_bk_name,in_rec_corr_acc,intermed_bank_acc,reserved_intermd_bk_ac_name,in_intermed_acc,instructed_amt,in_intermed_bk,in_exch_rate,rate_fixing,cover_method,in_3rd_reimb_bk,in_3rd_reimb_acc,mt103_type,extend_format,extend_info,inw_send_bic,in_send_chg,in_rec_chg,ac_chg_req_id,tot_rec_chg_crccy,tot_snd_chg_crccy,chg_advice_msg,expected_cover_id,netting_status,auth_date,bk_operation_code,am_inflow_rate,rate_fixing_ind,parent_txn_id,in_debit_acct_no,in_ben_our_charges,in_debit_vdate,in_intmed_bank,round_type,beneficiary_id,msg_type,msg_date,signatory,card_number,card_txn_detail,return_cheque,in_c_ord_bk,in_c_intmed_bk,in_c_acc_wit_bk,in_c_bk_t_bk_in,tfs_reference,cheque_drawn,ord_cust_code,issue_cheque_type,stock_number,payee_name,stock_register,series_id,ord_cust_acct,reserved_ord_cust_ac_name,in_ord_cust_acct,iban_debit,iban_credit,iban_ben,iban_acct_with_bank,iban_rec_corr_bank,iban_intermed_bank,iban_ord_cust_acc,iban_charges_acct,bic_iban_ben,bic_iban_ben_name,bic_iban_ben_city,ben_name,ben_address,ben_country,ben_town,in_ben_name,in_ben_address,in_ben_country,in_ben_town,ref_data_item,ref_data_value,loc_equiv_chgs,for_equiv_chgs,cb_credit_account,cb_debit_ccy,cb_credit_ccy,cb_debit_account,reg_compliance,tc_sim_accepted,last_sim_time,reserved_4,reserved_3,reserved_2,split_exposure_date,split_exposure_amt,in_hdr_3_uetr,authorisation_id,exempt_from_sca,status_reason_code,ft1,iblc_code_cr,iblc_code_ch,iblc_country_dr,iblc_country_cr,iblc_country_ch,future_date,iblc_code_dr,immediately,last_version,otor_version,dr_narrative,cr_narrative,l_va_number,card_no,recipt_no,item_id,inter_acct_no,inter_acct_name,inter_bank_id,inter_type,inter_bank_name,skn_flag,skn_status,skn_trx_type,skn_snd_clrg_cd,skn_otdb_exp_dt,skn_otdb_cu_acc,skn_otdb_dr_chq,skn_otdb_dr_acc,skn_otdb_dr_amt,skn_otdb_txn_cd,skn_otdb_bnk_cd,skn_otdb_city,skn_otdb_prov,skn_otdb_kbi,skn_otdb_region,skn_otdb_bs_typ,skn_otdb_totamt,skn_otcr_ben_ac,skn_otcr_ben_nm,skn_otcr_ben_rs,skn_otcr_ben_nt,skn_otcr_txn_cd,skn_otcr_bnk_cd,skn_otcr_bs_typ,skn_otcr_city,skn_otcr_prov,skn_otcr_kbi,skn_otcr_region,skn_otcr_snd_nm,skn_otcr_remark,rtgs_flag,rtgs_status,rtgs_trx_type,rtgs_trn_code,rtgs_trn_type,rtgs_rel_trn,rtgs_sen_mem_cd,rtgs_sen_mem_ac,rtgs_sen_mem_nm,rtgs_ben_ult_ac,rtgs_ben_ult_nm,rtgs_mem_info,rtgs_sen_ori_ac,rtgs_sen_ori_nm,rtgs_ben_mem_cd,rtgs_ben_mem_ac,rtgs_ben_mem_nm,skn_receiv_bank,skn_txn_code,draw_chq_no,skn_sor,skn_vlds_code,skn_vlds_rmrk,skn_data_id,skn_ofs_err_msg,skn_ben_clrg_cd,l_ben_name,related_trn,skn_sender_name,resident_status,nation_status,rtgs_data_id,cur_dhn_flag,skn_data_id_ret,l_cashback_ref,alt_card_id,sett_stmt_id,l_ft_type,skn_otcr_ctype,skn_sti,skn_bulk,skn_bulk_reff,skn_otdb_idpel,skn_otdb_name,sort_cust_acct,sort_draw_chq,sort_txn_code,sort_rcv_bnk_cd,sort_rcv_clr_cd,sort_rcv_city,sort_rcv_prov,sort_rcv_kbi,sort_rcv_region,sort_rcv_bustyp,sort_reason_cd,sort_cust_name,sort_cust_type,sort_tax_reg_no,sort_birth_date,sort_snd_name,sort_address,sort_rt,sort_rw,sort_post_code,sort_prov_state,sort_city_mncpl,sort_chrg_waive,sort_chrg_code,batch_file_id,pdc_flag,rtgs_remark,prk_limit_id,customer_charge,l_mm_fasbi_ref,fasbi_prin,bil_product,bil_customer,bl_cust_no,bl_product,bl_code,l_bl_tokens,bl_cust_name,mcc_type,mcc_desc,l_sp_rate_id,lhbu_generated,fx_purpose,fx_document,fx_doc_desc,chn_sender_nm,agent_code,agent_desc,agent_location,mcc_code,merchant_id,merchant_loc,l_teller_id,reason_code,l_sms_1,l_phone_1,sid_dati2_lahir,kyc_incom_rng,kyc_sumber_dana,ktp_provinsi,ktp_kelurahan,ktp_kecamatan,wic_nationality,kd_kelurahan,l_pep,wic_negara_addr,wic_dati_ii_dom,wic_prov_dom,wic_negara_dom,l_wic_id,l_teller_acc,l_spmu_id,l_spmu_no,l_asset_reg_fa,l_ft_sewa,snd_pnbrek_pem,snd_pngrek_pem,onus_cr_acct_nm,onus_dr_acct_nm,va_acct_name,term_type,term_id,trans_reff,term_narr,teller_ref,to_acct_no,update_time,upd_dhn_flag,wic_marital_sts,wic_ind_birthdt,wic_occptn,wic_post_code,wic_address_id,wic_address_cur,wic_jenis_kelam,wic_npwp,wic_rt,wic_rw,wic_ind_birthpl,wic_cust_type,wic_cor_birthdt,wic_cor_birthpl,wic_nama_alias,wic_kerja_lain,wic_bid_usaha,wic_7an_trx,wic_address_dom,wic_kel_dom,wic_kec_dom,wic_kd_pos_dom,wic_flag,wic_full_name,wic_id_type,wic_id_no,wic_cor_name,wic_cor_address,tgl_warkat,bif_cat_purpose,bif_crd_rsdn_st,bif_crd_town_cd,bif_crd_type,bif_creat_dt_tm,bif_cred_acc_no,bif_cred_acc_tp,bif_cred_name,bif_cred_nik,bif_cred_pxy_tp,bif_cred_pxy_vl,bif_dbt_acc_tp,bif_dbt_name,bif_dbt_nik,bif_dbt_rsdn_st,bif_dbt_town_cd,bif_dbt_type,bif_orig_bic,bif_recv_bic,bif_direction,bif_dbt_acc_no,l_rtgs_chg_code,lld_benef_type,benef_country,lld_payer_type,payer_country,rel_btw_party,txn_country,lld_txn_purpose,l_manual_risk,flag_non_pinpad,dest_bank_id,l_slip_no,l_acct_sender,lld_info_doc,l_acct_receiver'; + + $fieldsArray = explode(',', $fields); + + Schema::table('temp_funds_transfer', function (Blueprint $table) use ($fieldsArray) { + foreach ($fieldsArray as $field) { + $field = trim($field); + // Change field type back to string from text + $table->string($field)->nullable()->change(); + } + }); + } + };