From e511025307f4a69322f38f5b667b926923cee63a Mon Sep 17 00:00:00 2001 From: Daeng Deni Mardaeni Date: Wed, 21 May 2025 22:01:31 +0700 Subject: [PATCH] feat(webstatement): tambahkan fitur proses data teller - Menambahkan job baru `ProcessTellerDataJob` untuk memproses data teller dari file CSV. - Membuat controller method `ProcessTellerData` di `MigrasiController` untuk dispatch job `ProcessTellerDataJob`. - Menambahkan model baru `Teller` yang merepresentasikan data teller dengan kolom sesuai header map pada CSV. - Menambahkan migrasi `2025_05_21_144332_create_tellers_table.php` untuk membuat tabel database `tellers` dengan struktur data lengkap. - Menambahkan logika pada job untuk membaca, memproses, dan menyimpan data CSV ke dalam database. - Menyediakan mekanisme pemrosesan file CSV termasuk: - Validasi jumlah kolom pada CSV jika tidak sesuai dengan header. - Membuat dan menghapus file sementara selama pemrosesan. - Mapping data CSV ke atribut model `Teller`. - Penanganan error saat menyimpan data ke model `Teller`. - Logging proses meliputi jumlah data yang diproses dan error selama jalannya job. Signed-off-by: Daeng Deni Mardaeni --- app/Http/Controllers/MigrasiController.php | 11 + app/Jobs/ProcessTellerDataJob.php | 316 ++++++++++++++++++ app/Models/Teller.php | 117 +++++++ ...2025_05_21_144332_create_tellers_table.php | 130 +++++++ 4 files changed, 574 insertions(+) create mode 100644 app/Jobs/ProcessTellerDataJob.php create mode 100644 app/Models/Teller.php create mode 100644 database/migrations/2025_05_21_144332_create_tellers_table.php diff --git a/app/Http/Controllers/MigrasiController.php b/app/Http/Controllers/MigrasiController.php index b11e030..1d46fcb 100644 --- a/app/Http/Controllers/MigrasiController.php +++ b/app/Http/Controllers/MigrasiController.php @@ -18,6 +18,7 @@ use Modules\Webstatement\Jobs\ProcessFundsTransferDataJob; use Modules\Webstatement\Jobs\ProcessStmtEntryDataJob; use Modules\Webstatement\Jobs\ProcessStmtNarrFormatDataJob; use Modules\Webstatement\Jobs\ProcessStmtNarrParamDataJob; +use Modules\Webstatement\Jobs\ProcessTellerDataJob; use Modules\Webstatement\Jobs\ProcessTransactionDataJob; class MigrasiController extends Controller @@ -152,6 +153,15 @@ class MigrasiController extends Controller } } + public function ProcessTellerData($periods){ + try { + ProcessTellerDataJob::dispatch($periods); + return response()->json(['message' => 'Data TempStmtEntry processing job has been successfully']); + } catch (Exception $e) { + return response()->json(['error' => $e->getMessage()], 500); + } + } + public function index() { @@ -187,6 +197,7 @@ class MigrasiController extends Controller //$this->processStmtEntryData($periods); //$this->ProcessDataCaptureData($periods); //$this->processFundsTransferData($periods); + $this->ProcessTellerData($periods); //$this->processArrangementData($periods); //$this->processBillDetailData($periods); diff --git a/app/Jobs/ProcessTellerDataJob.php b/app/Jobs/ProcessTellerDataJob.php new file mode 100644 index 0000000..0526ffd --- /dev/null +++ b/app/Jobs/ProcessTellerDataJob.php @@ -0,0 +1,316 @@ + 'id_teller', + 'account_1' => 'account_1', + 'currency_1' => 'currency_1', + 'amount_local_1' => 'amount_local_1', + 'value_date_1' => 'value_date_1', + 'account_2' => 'account_2', + 'new_cust_bal' => 'new_cust_bal', + 'term_type' => 'term_type', + 'term_id' => 'term_id', + 'trans_reff' => 'trans_reff', + 'card_no' => 'card_no', + 'recipt_no' => 'recipt_no', + 'transaction_code' => 'transaction_code', + 'date_time' => 'date_time', + 'record_status' => 'record_status', + 'amount_local_2' => 'amount_local_2', + 'co_code' => 'co_code', + 'narrative_1' => 'narrative_1', + 'wic_flag' => 'wic_flag', + 'wic_cust_type' => 'wic_cust_type', + 'wic_full_name' => 'wic_full_name', + 'wic_alias_name' => 'wic_alias_name', + 'wic_acct_no' => 'wic_acct_no', + 'wic_id_type' => 'wic_id_type', + 'wic_id_no' => 'wic_id_no', + 'wic_npwp' => 'wic_npwp', + 'wic_nationality' => 'wic_nationality', + 'wic_ind_birthpl' => 'wic_ind_birthpl', + 'wic_ind_birthdt' => 'wic_ind_birthdt', + 'wic_address_id' => 'wic_address_id', + 'wic_address_cur' => 'wic_address_cur', + 'wic_city' => 'wic_city', + 'wic_province' => 'wic_province', + 'wic_post_code' => 'wic_post_code', + 'wic_phone' => 'wic_phone', + 'wic_gender' => 'wic_gender', + 'wic_marital_sts' => 'wic_marital_sts', + 'wic_occptn' => 'wic_occptn', + 'wic_occptn_dur' => 'wic_occptn_dur', + 'wic_income_avg' => 'wic_income_avg', + 'wic_cor_name' => 'wic_cor_name', + 'wic_cor_address' => 'wic_cor_address', + 'wic_cor_phone' => 'wic_cor_phone', + 'wic_cor_lgl_typ' => 'wic_cor_lgl_typ', + 'wic_cor_lic_no' => 'wic_cor_lic_no', + 'wic_cor_birthpl' => 'wic_cor_birthpl', + 'wic_cor_birthdt' => 'wic_cor_birthdt', + 'wic_cor_rel' => 'wic_cor_rel', + 'wic_party_rel' => 'wic_party_rel', + 'wic_amount' => 'wic_amount', + 'wic_amount_type' => 'wic_amount_type', + 'wic_amtbnk_name' => 'wic_amtbnk_name', + 'wic_amtbnk_cunm' => 'wic_amtbnk_cunm', + 'wic_fund_source' => 'wic_fund_source', + 'wic_fund_use' => 'wic_fund_use', + 'dr_cr_marker' => 'dr_cr_marker', + 'charge_code' => 'charge_code', + 'chrg_amt_local' => 'chrg_amt_local', + 'charge_category' => 'charge_category', + 'charge_account' => 'charge_account', + 'amount_fcy_1' => 'amount_fcy_1', + 'rate_1' => 'rate_1', + 'deal_rate' => 'deal_rate', + 'l_wic_id' => 'l_wic_id', + 'account_1_co_code' => 'account_1_co_code', + 'account_2_co_code' => 'account_2_co_code', + 'l_charge_amt' => 'l_charge_amt', + 'bl_cust_no' => 'bl_cust_no', + 'stmt_no' => 'stmt_no', + 'bil_customer' => 'bil_customer', + 'value_date_2' => 'value_date_2', + 'cheq_type' => 'cheq_type', + 'cheque_number' => 'cheque_number', + 'inputter' => 'inputter', + 'authoriser' => 'authoriser', + 'bil_product' => 'bil_product', + 'fx_document' => 'fx_document', + 'fx_purpose' => 'fx_purpose', + 'narrative_2' => 'narrative_2', + 'customer_2' => 'customer_2', + 'l_sms_1' => 'l_sms_1', + 'l_phone_1' => 'l_phone_1', + 'kyc_incom_rng' => 'kyc_incom_rng', + 'wic_rt' => 'wic_rt', + 'wic_rw' => 'wic_rw', + 'ktp_kelurahan' => 'ktp_kelurahan', + 'ktp_kecamatan' => 'ktp_kecamatan', + 'ktp_provinsi' => 'ktp_provinsi', + 'wic_jenis_kelam' => 'wic_jenis_kelam', + 'kyc_sumber_dana' => 'kyc_sumber_dana', + 'l_manual_risk' => 'l_manual_risk', + 'l_slip_no' => 'l_slip_no', + 'dest_bank_id' => 'dest_bank_id', + 'dr_narrative' => 'dr_narrative', + 'inter_type' => 'inter_type', + 'l_va_number' => 'l_va_number', + 'inter_bank_id' => 'inter_bank_id', + 'term_narr' => 'term_narr', + 'currency_2' => 'currency_2', + 'amount_fcy_2' => 'amount_fcy_2', + 'rate_2' => 'rate_2', + 'customer_1' => 'customer_1', + 'last_version' => 'last_version' + ]; + + // Pemetaan bidang header ke kolom model + protected array $periods; + + /** + * Create a new job instance. + */ + public function __construct(array $periods = []) + { + $this->periods = $periods; + } + + /** + * Execute the job. + */ + public function handle() + : void + { + try { + set_time_limit(24 * 60 * 60); + + if (empty($this->periods)) { + Log::warning('No periods provided for teller data processing'); + return; + } + + $stats = $this->processPeriods(); + + Log::info("ProcessTellerDataJob completed. Total processed: {$stats['processed']}, Total errors: {$stats['errors']}"); + } catch (Exception $e) { + Log::error("Error in ProcessTellerDataJob: " . $e->getMessage()); + throw $e; + } + } + + /** + * Process all periods and return statistics + */ + private function processPeriods() + : array + { + $disk = Storage::disk(self::DISK_NAME); + $processedCount = 0; + $errorCount = 0; + + foreach ($this->periods as $period) { + // Skip the parameter folder + if ($period === self::PARAMETER_FOLDER) { + Log::info("Skipping " . self::PARAMETER_FOLDER . " folder"); + continue; + } + + $result = $this->processPeriodFile($disk, $period); + $processedCount += $result['processed']; + $errorCount += $result['errors']; + } + + return [ + 'processed' => $processedCount, + 'errors' => $errorCount + ]; + } + + /** + * Process a single period file + */ + private function processPeriodFile($disk, string $period) + : array + { + $filename = $period . self::FILE_EXTENSION; + $filePath = "$period/$filename"; + $processedCount = 0; + $errorCount = 0; + + Log::info("Processing teller file: $filePath"); + + if (!$disk->exists($filePath)) { + Log::warning("File not found: $filePath"); + return ['processed' => 0, 'errors' => 0]; + } + + $tempFilePath = $this->createTempFile($disk, $filePath, $filename); + + $result = $this->processCSVFile($tempFilePath, $filePath); + $processedCount += $result['processed']; + $errorCount += $result['errors']; + + // Clean up the temporary file + if (file_exists($tempFilePath)) { + unlink($tempFilePath); + } + + Log::info("Completed processing $filePath. Processed {$result['processed']} records with {$result['errors']} errors."); + + return [ + 'processed' => $processedCount, + 'errors' => $errorCount + ]; + } + + /** + * Create a temporary file for processing + */ + private function createTempFile($disk, string $filePath, string $filename) + : string + { + $tempFilePath = storage_path("app/temp_$filename"); + file_put_contents($tempFilePath, $disk->get($filePath)); + return $tempFilePath; + } + + /** + * Process a CSV file and import data + */ + private function processCSVFile(string $tempFilePath, string $originalFilePath) + : array + { + $processedCount = 0; + $errorCount = 0; + + $handle = fopen($tempFilePath, "r"); + if ($handle === false) { + Log::error("Unable to open file: $originalFilePath"); + return ['processed' => 0, 'errors' => 0]; + } + + // Get the headers from the first row + $headerRow = fgetcsv($handle, 0, self::CSV_DELIMITER); + if (!$headerRow) { + fclose($handle); + return ['processed' => 0, 'errors' => 0]; + } + + $rowCount = 0; + while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) { + $rowCount++; + + if (count($headerRow) !== count($row)) { + Log::warning("Row $rowCount in $originalFilePath has incorrect column count. Expected: " . count($headerRow) . ", Got: " . count($row)); + continue; + } + + $result = $this->processRow($headerRow, $row, $rowCount, $originalFilePath); + $processedCount += $result['processed']; + $errorCount += $result['errors']; + } + + fclose($handle); + + return [ + 'processed' => $processedCount, + 'errors' => $errorCount + ]; + } + + /** + * Process a single row from the CSV file + */ + private function processRow(array $headerRow, array $row, int $rowCount, string $filePath) + : array + { + // Combine the header row with the data row + $rawData = array_combine($headerRow, $row); + + // Map the raw data to our model fields + $data = []; + foreach (self::HEADER_MAP as $csvField => $modelField) { + $data[$modelField] = $rawData[$csvField] ?? null; + } + + // Skip header row if it was included in the data + if ($data['id_teller'] === 'id') { + return ['processed' => 0, 'errors' => 0]; + } + + try { + $teller = Teller::firstOrNew(['id_teller' => $data['id_teller']]); + $teller->fill($data); + $teller->save(); + + return ['processed' => 1, 'errors' => 0]; + } catch (Exception $e) { + Log::error("Error processing Teller at row $rowCount in $filePath: " . $e->getMessage()); + return ['processed' => 0, 'errors' => 1]; + } + } + } diff --git a/app/Models/Teller.php b/app/Models/Teller.php new file mode 100644 index 0000000..26d8cfa --- /dev/null +++ b/app/Models/Teller.php @@ -0,0 +1,117 @@ +id(); + $table->string('id_teller')->nullable(); + $table->string('account_1')->nullable(); + $table->string('currency_1')->nullable(); + $table->string('amount_local_1')->nullable(); + $table->string('value_date_1')->nullable(); + $table->string('account_2')->nullable(); + $table->string('new_cust_bal')->nullable(); + $table->string('term_type')->nullable(); + $table->string('term_id')->nullable(); + $table->string('trans_reff')->nullable(); + $table->string('card_no')->nullable(); + $table->string('recipt_no')->nullable(); + $table->string('transaction_code')->nullable(); + $table->string('date_time')->nullable(); + $table->string('record_status')->nullable(); + $table->string('amount_local_2')->nullable(); + $table->string('co_code')->nullable(); + $table->text('narrative_1')->nullable(); + $table->string('wic_flag')->nullable(); + $table->string('wic_cust_type')->nullable(); + $table->string('wic_full_name')->nullable(); + $table->string('wic_alias_name')->nullable(); + $table->string('wic_acct_no')->nullable(); + $table->string('wic_id_type')->nullable(); + $table->string('wic_id_no')->nullable(); + $table->string('wic_npwp')->nullable(); + $table->string('wic_nationality')->nullable(); + $table->string('wic_ind_birthpl')->nullable(); + $table->string('wic_ind_birthdt')->nullable(); + $table->string('wic_address_id')->nullable(); + $table->string('wic_address_cur')->nullable(); + $table->string('wic_city')->nullable(); + $table->string('wic_province')->nullable(); + $table->string('wic_post_code')->nullable(); + $table->string('wic_phone')->nullable(); + $table->string('wic_gender')->nullable(); + $table->string('wic_marital_sts')->nullable(); + $table->string('wic_occptn')->nullable(); + $table->string('wic_occptn_dur')->nullable(); + $table->string('wic_income_avg')->nullable(); + $table->string('wic_cor_name')->nullable(); + $table->string('wic_cor_address')->nullable(); + $table->string('wic_cor_phone')->nullable(); + $table->string('wic_cor_lgl_typ')->nullable(); + $table->string('wic_cor_lic_no')->nullable(); + $table->string('wic_cor_birthpl')->nullable(); + $table->string('wic_cor_birthdt')->nullable(); + $table->string('wic_cor_rel')->nullable(); + $table->string('wic_party_rel')->nullable(); + $table->string('wic_amount')->nullable(); + $table->string('wic_amount_type')->nullable(); + $table->string('wic_amtbnk_name')->nullable(); + $table->string('wic_amtbnk_cunm')->nullable(); + $table->string('wic_fund_source')->nullable(); + $table->string('wic_fund_use')->nullable(); + $table->string('dr_cr_marker')->nullable(); + $table->string('charge_code')->nullable(); + $table->string('chrg_amt_local')->nullable(); + $table->string('charge_category')->nullable(); + $table->string('charge_account')->nullable(); + $table->string('amount_fcy_1')->nullable(); + $table->string('rate_1')->nullable(); + $table->string('deal_rate')->nullable(); + $table->string('l_wic_id')->nullable(); + $table->string('account_1_co_code')->nullable(); + $table->string('account_2_co_code')->nullable(); + $table->string('l_charge_amt')->nullable(); + $table->string('bl_cust_no')->nullable(); + $table->string('stmt_no')->nullable(); + $table->string('bil_customer')->nullable(); + $table->string('value_date_2')->nullable(); + $table->string('cheq_type')->nullable(); + $table->string('cheque_number')->nullable(); + $table->string('inputter')->nullable(); + $table->string('authoriser')->nullable(); + $table->string('bil_product')->nullable(); + $table->string('fx_document')->nullable(); + $table->string('fx_purpose')->nullable(); + $table->text('narrative_2')->nullable(); + $table->string('customer_2')->nullable(); + $table->string('l_sms_1')->nullable(); + $table->string('l_phone_1')->nullable(); + $table->string('kyc_incom_rng')->nullable(); + $table->string('wic_rt')->nullable(); + $table->string('wic_rw')->nullable(); + $table->string('ktp_kelurahan')->nullable(); + $table->string('ktp_kecamatan')->nullable(); + $table->string('ktp_provinsi')->nullable(); + $table->string('wic_jenis_kelam')->nullable(); + $table->string('kyc_sumber_dana')->nullable(); + $table->string('l_manual_risk')->nullable(); + $table->string('l_slip_no')->nullable(); + $table->string('dest_bank_id')->nullable(); + $table->text('dr_narrative')->nullable(); + $table->string('inter_type')->nullable(); + $table->string('l_va_number')->nullable(); + $table->string('inter_bank_id')->nullable(); + $table->text('term_narr')->nullable(); + $table->string('currency_2')->nullable(); + $table->string('amount_fcy_2')->nullable(); + $table->string('rate_2')->nullable(); + $table->string('customer_1')->nullable(); + $table->string('last_version')->nullable(); + $table->timestamps(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::dropIfExists('tellers'); + } + };