daengdeni/webstatement
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(webstatement): tambahkan fitur proses data teller

Browse Source 
- Menambahkan job baru `ProcessTellerDataJob` untuk memproses data teller dari file CSV.
- Membuat controller method `ProcessTellerData` di `MigrasiController` untuk dispatch job `ProcessTellerDataJob`.
- Menambahkan model baru `Teller` yang merepresentasikan data teller dengan kolom sesuai header map pada CSV.
- Menambahkan migrasi `2025_05_21_144332_create_tellers_table.php` untuk membuat tabel database `tellers` dengan struktur data lengkap.
- Menambahkan logika pada job untuk membaca, memproses, dan menyimpan data CSV ke dalam database.
- Menyediakan mekanisme pemrosesan file CSV termasuk:
  - Validasi jumlah kolom pada CSV jika tidak sesuai dengan header.
  - Membuat dan menghapus file sementara selama pemrosesan.
  - Mapping data CSV ke atribut model `Teller`.
  - Penanganan error saat menyimpan data ke model `Teller`.
  - Logging proses meliputi jumlah data yang diproses dan error selama jalannya job.

Signed-off-by: Daeng Deni Mardaeni <ddeni05@gmail.com>
This commit is contained in:
Daeng Deni Mardaeni
2025-05-21 22:01:31 +07:00
parent 285c2409ea
commit e511025307
4 changed files with 574 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
app/Http/Controllers/MigrasiController.php
View File

@@ -18,6 +18,7 @@ use Modules\Webstatement\Jobs\ProcessFundsTransferDataJob;
use Modules\Webstatement\Jobs\ProcessStmtEntryDataJob;
use Modules\Webstatement\Jobs\ProcessStmtNarrFormatDataJob;
use Modules\Webstatement\Jobs\ProcessStmtNarrParamDataJob;
use Modules\Webstatement\Jobs\ProcessTellerDataJob;
use Modules\Webstatement\Jobs\ProcessTransactionDataJob;
class MigrasiController extends Controller
@@ -152,6 +153,15 @@ class MigrasiController extends Controller
}
}
public function ProcessTellerData($periods){
try {
ProcessTellerDataJob::dispatch($periods);
return response()->json(['message' => 'Data TempStmtEntry processing job has been successfully']);
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function index()
{
@@ -187,6 +197,7 @@ class MigrasiController extends Controller
//$this->processStmtEntryData($periods);
//$this->ProcessDataCaptureData($periods);
//$this->processFundsTransferData($periods);
$this->ProcessTellerData($periods);
//$this->processArrangementData($periods);
//$this->processBillDetailData($periods);

316
app/Jobs/ProcessTellerDataJob.php Normal file
View File

@@ -0,0 +1,316 @@
<?php
namespace Modules\Webstatement\Jobs;
use Exception;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
use Modules\Webstatement\Models\Teller;
class ProcessTellerDataJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
private const PARAMETER_FOLDER = '_parameter';
// Konstanta untuk nilai-nilai statis
private const FILE_EXTENSION = '.ST.TELLER.csv';
private const CSV_DELIMITER = '~';
private const DISK_NAME = 'sftpStatement';
private const HEADER_MAP = [
'id' => 'id_teller',
'account_1' => 'account_1',
'currency_1' => 'currency_1',
'amount_local_1' => 'amount_local_1',
'value_date_1' => 'value_date_1',
'account_2' => 'account_2',
'new_cust_bal' => 'new_cust_bal',
'term_type' => 'term_type',
'term_id' => 'term_id',
'trans_reff' => 'trans_reff',
'card_no' => 'card_no',
'recipt_no' => 'recipt_no',
'transaction_code' => 'transaction_code',
'date_time' => 'date_time',
'record_status' => 'record_status',
'amount_local_2' => 'amount_local_2',
'co_code' => 'co_code',
'narrative_1' => 'narrative_1',
'wic_flag' => 'wic_flag',
'wic_cust_type' => 'wic_cust_type',
'wic_full_name' => 'wic_full_name',
'wic_alias_name' => 'wic_alias_name',
'wic_acct_no' => 'wic_acct_no',
'wic_id_type' => 'wic_id_type',
'wic_id_no' => 'wic_id_no',
'wic_npwp' => 'wic_npwp',
'wic_nationality' => 'wic_nationality',
'wic_ind_birthpl' => 'wic_ind_birthpl',
'wic_ind_birthdt' => 'wic_ind_birthdt',
'wic_address_id' => 'wic_address_id',
'wic_address_cur' => 'wic_address_cur',
'wic_city' => 'wic_city',
'wic_province' => 'wic_province',
'wic_post_code' => 'wic_post_code',
'wic_phone' => 'wic_phone',
'wic_gender' => 'wic_gender',
'wic_marital_sts' => 'wic_marital_sts',
'wic_occptn' => 'wic_occptn',
'wic_occptn_dur' => 'wic_occptn_dur',
'wic_income_avg' => 'wic_income_avg',
'wic_cor_name' => 'wic_cor_name',
'wic_cor_address' => 'wic_cor_address',
'wic_cor_phone' => 'wic_cor_phone',
'wic_cor_lgl_typ' => 'wic_cor_lgl_typ',
'wic_cor_lic_no' => 'wic_cor_lic_no',
'wic_cor_birthpl' => 'wic_cor_birthpl',
'wic_cor_birthdt' => 'wic_cor_birthdt',
'wic_cor_rel' => 'wic_cor_rel',
'wic_party_rel' => 'wic_party_rel',
'wic_amount' => 'wic_amount',
'wic_amount_type' => 'wic_amount_type',
'wic_amtbnk_name' => 'wic_amtbnk_name',
'wic_amtbnk_cunm' => 'wic_amtbnk_cunm',
'wic_fund_source' => 'wic_fund_source',
'wic_fund_use' => 'wic_fund_use',
'dr_cr_marker' => 'dr_cr_marker',
'charge_code' => 'charge_code',
'chrg_amt_local' => 'chrg_amt_local',
'charge_category' => 'charge_category',
'charge_account' => 'charge_account',
'amount_fcy_1' => 'amount_fcy_1',
'rate_1' => 'rate_1',
'deal_rate' => 'deal_rate',
'l_wic_id' => 'l_wic_id',
'account_1_co_code' => 'account_1_co_code',
'account_2_co_code' => 'account_2_co_code',
'l_charge_amt' => 'l_charge_amt',
'bl_cust_no' => 'bl_cust_no',
'stmt_no' => 'stmt_no',
'bil_customer' => 'bil_customer',
'value_date_2' => 'value_date_2',
'cheq_type' => 'cheq_type',
'cheque_number' => 'cheque_number',
'inputter' => 'inputter',
'authoriser' => 'authoriser',
'bil_product' => 'bil_product',
'fx_document' => 'fx_document',
'fx_purpose' => 'fx_purpose',
'narrative_2' => 'narrative_2',
'customer_2' => 'customer_2',
'l_sms_1' => 'l_sms_1',
'l_phone_1' => 'l_phone_1',
'kyc_incom_rng' => 'kyc_incom_rng',
'wic_rt' => 'wic_rt',
'wic_rw' => 'wic_rw',
'ktp_kelurahan' => 'ktp_kelurahan',
'ktp_kecamatan' => 'ktp_kecamatan',
'ktp_provinsi' => 'ktp_provinsi',
'wic_jenis_kelam' => 'wic_jenis_kelam',
'kyc_sumber_dana' => 'kyc_sumber_dana',
'l_manual_risk' => 'l_manual_risk',
'l_slip_no' => 'l_slip_no',
'dest_bank_id' => 'dest_bank_id',
'dr_narrative' => 'dr_narrative',
'inter_type' => 'inter_type',
'l_va_number' => 'l_va_number',
'inter_bank_id' => 'inter_bank_id',
'term_narr' => 'term_narr',
'currency_2' => 'currency_2',
'amount_fcy_2' => 'amount_fcy_2',
'rate_2' => 'rate_2',
'customer_1' => 'customer_1',
'last_version' => 'last_version'
];
// Pemetaan bidang header ke kolom model
protected array $periods;
/**
* Create a new job instance.
*/
public function __construct(array $periods = [])
{
$this->periods = $periods;
}
/**
* Execute the job.
*/
public function handle()
: void
{
try {
set_time_limit(24 * 60 * 60);
if (empty($this->periods)) {
Log::warning('No periods provided for teller data processing');
return;
}
$stats = $this->processPeriods();
Log::info("ProcessTellerDataJob completed. Total processed: {$stats['processed']}, Total errors: {$stats['errors']}");
} catch (Exception $e) {
Log::error("Error in ProcessTellerDataJob: " . $e->getMessage());
throw $e;
}
}
/**
* Process all periods and return statistics
*/
private function processPeriods()
: array
{
$disk = Storage::disk(self::DISK_NAME);
$processedCount = 0;
$errorCount = 0;
foreach ($this->periods as $period) {
// Skip the parameter folder
if ($period === self::PARAMETER_FOLDER) {
Log::info("Skipping " . self::PARAMETER_FOLDER . " folder");
continue;
}
$result = $this->processPeriodFile($disk, $period);
$processedCount += $result['processed'];
$errorCount += $result['errors'];
}
return [
'processed' => $processedCount,
'errors' => $errorCount
];
}
/**
* Process a single period file
*/
private function processPeriodFile($disk, string $period)
: array
{
$filename = $period . self::FILE_EXTENSION;
$filePath = "$period/$filename";
$processedCount = 0;
$errorCount = 0;
Log::info("Processing teller file: $filePath");
if (!$disk->exists($filePath)) {
Log::warning("File not found: $filePath");
return ['processed' => 0, 'errors' => 0];
}
$tempFilePath = $this->createTempFile($disk, $filePath, $filename);
$result = $this->processCSVFile($tempFilePath, $filePath);
$processedCount += $result['processed'];
$errorCount += $result['errors'];
// Clean up the temporary file
if (file_exists($tempFilePath)) {
unlink($tempFilePath);
}
Log::info("Completed processing $filePath. Processed {$result['processed']} records with {$result['errors']} errors.");
return [
'processed' => $processedCount,
'errors' => $errorCount
];
}
/**
* Create a temporary file for processing
*/
private function createTempFile($disk, string $filePath, string $filename)
: string
{
$tempFilePath = storage_path("app/temp_$filename");
file_put_contents($tempFilePath, $disk->get($filePath));
return $tempFilePath;
}
/**
* Process a CSV file and import data
*/
private function processCSVFile(string $tempFilePath, string $originalFilePath)
: array
{
$processedCount = 0;
$errorCount = 0;
$handle = fopen($tempFilePath, "r");
if ($handle === false) {
Log::error("Unable to open file: $originalFilePath");
return ['processed' => 0, 'errors' => 0];
}
// Get the headers from the first row
$headerRow = fgetcsv($handle, 0, self::CSV_DELIMITER);
if (!$headerRow) {
fclose($handle);
return ['processed' => 0, 'errors' => 0];
}
$rowCount = 0;
while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) {
$rowCount++;
if (count($headerRow) !== count($row)) {
Log::warning("Row $rowCount in $originalFilePath has incorrect column count. Expected: " . count($headerRow) . ", Got: " . count($row));
continue;
}
$result = $this->processRow($headerRow, $row, $rowCount, $originalFilePath);
$processedCount += $result['processed'];
$errorCount += $result['errors'];
}
fclose($handle);
return [
'processed' => $processedCount,
'errors' => $errorCount
];
}
/**
* Process a single row from the CSV file
*/
private function processRow(array $headerRow, array $row, int $rowCount, string $filePath)
: array
{
// Combine the header row with the data row
$rawData = array_combine($headerRow, $row);
// Map the raw data to our model fields
$data = [];
foreach (self::HEADER_MAP as $csvField => $modelField) {
$data[$modelField] = $rawData[$csvField] ?? null;
}
// Skip header row if it was included in the data
if ($data['id_teller'] === 'id') {
return ['processed' => 0, 'errors' => 0];
}
try {
$teller = Teller::firstOrNew(['id_teller' => $data['id_teller']]);
$teller->fill($data);
$teller->save();
return ['processed' => 1, 'errors' => 0];
} catch (Exception $e) {
Log::error("Error processing Teller at row $rowCount in $filePath: " . $e->getMessage());
return ['processed' => 0, 'errors' => 1];
}
}
}

117
app/Models/Teller.php Normal file
View File

@@ -0,0 +1,117 @@
<?php
namespace Modules\Webstatement\Models;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\Factories\HasFactory;
class Teller extends Model
{
use HasFactory;
protected $fillable = [
'id_teller',
'account_1',
'currency_1',
'amount_local_1',
'value_date_1',
'account_2',
'new_cust_bal',
'term_type',
'term_id',
'trans_reff',
'card_no',
'recipt_no',
'transaction_code',
'date_time',
'record_status',
'amount_local_2',
'co_code',
'narrative_1',
'wic_flag',
'wic_cust_type',
'wic_full_name',
'wic_alias_name',
'wic_acct_no',
'wic_id_type',
'wic_id_no',
'wic_npwp',
'wic_nationality',
'wic_ind_birthpl',
'wic_ind_birthdt',
'wic_address_id',
'wic_address_cur',
'wic_city',
'wic_province',
'wic_post_code',
'wic_phone',
'wic_gender',
'wic_marital_sts',
'wic_occptn',
'wic_occptn_dur',
'wic_income_avg',
'wic_cor_name',
'wic_cor_address',
'wic_cor_phone',
'wic_cor_lgl_typ',
'wic_cor_lic_no',
'wic_cor_birthpl',
'wic_cor_birthdt',
'wic_cor_rel',
'wic_party_rel',
'wic_amount',
'wic_amount_type',
'wic_amtbnk_name',
'wic_amtbnk_cunm',
'wic_fund_source',
'wic_fund_use',
'dr_cr_marker',
'charge_code',
'chrg_amt_local',
'charge_category',
'charge_account',
'amount_fcy_1',
'rate_1',
'deal_rate',
'l_wic_id',
'account_1_co_code',
'account_2_co_code',
'l_charge_amt',
'bl_cust_no',
'stmt_no',
'bil_customer',
'value_date_2',
'cheq_type',
'cheque_number',
'inputter',
'authoriser',
'bil_product',
'fx_document',
'fx_purpose',
'narrative_2',
'customer_2',
'l_sms_1',
'l_phone_1',
'kyc_incom_rng',
'wic_rt',
'wic_rw',
'ktp_kelurahan',
'ktp_kecamatan',
'ktp_provinsi',
'wic_jenis_kelam',
'kyc_sumber_dana',
'l_manual_risk',
'l_slip_no',
'dest_bank_id',
'dr_narrative',
'inter_type',
'l_va_number',
'inter_bank_id',
'term_narr',
'currency_2',
'amount_fcy_2',
'rate_2',
'customer_1',
'last_version'
];
}

130
database/migrations/2025_05_21_144332_create_tellers_table.php Normal file
View File

@@ -0,0 +1,130 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::create('tellers', function (Blueprint $table) {
$table->id();
$table->string('id_teller')->nullable();
$table->string('account_1')->nullable();
$table->string('currency_1')->nullable();
$table->string('amount_local_1')->nullable();
$table->string('value_date_1')->nullable();
$table->string('account_2')->nullable();
$table->string('new_cust_bal')->nullable();
$table->string('term_type')->nullable();
$table->string('term_id')->nullable();
$table->string('trans_reff')->nullable();
$table->string('card_no')->nullable();
$table->string('recipt_no')->nullable();
$table->string('transaction_code')->nullable();
$table->string('date_time')->nullable();
$table->string('record_status')->nullable();
$table->string('amount_local_2')->nullable();
$table->string('co_code')->nullable();
$table->text('narrative_1')->nullable();
$table->string('wic_flag')->nullable();
$table->string('wic_cust_type')->nullable();
$table->string('wic_full_name')->nullable();
$table->string('wic_alias_name')->nullable();
$table->string('wic_acct_no')->nullable();
$table->string('wic_id_type')->nullable();
$table->string('wic_id_no')->nullable();
$table->string('wic_npwp')->nullable();
$table->string('wic_nationality')->nullable();
$table->string('wic_ind_birthpl')->nullable();
$table->string('wic_ind_birthdt')->nullable();
$table->string('wic_address_id')->nullable();
$table->string('wic_address_cur')->nullable();
$table->string('wic_city')->nullable();
$table->string('wic_province')->nullable();
$table->string('wic_post_code')->nullable();
$table->string('wic_phone')->nullable();
$table->string('wic_gender')->nullable();
$table->string('wic_marital_sts')->nullable();
$table->string('wic_occptn')->nullable();
$table->string('wic_occptn_dur')->nullable();
$table->string('wic_income_avg')->nullable();
$table->string('wic_cor_name')->nullable();
$table->string('wic_cor_address')->nullable();
$table->string('wic_cor_phone')->nullable();
$table->string('wic_cor_lgl_typ')->nullable();
$table->string('wic_cor_lic_no')->nullable();
$table->string('wic_cor_birthpl')->nullable();
$table->string('wic_cor_birthdt')->nullable();
$table->string('wic_cor_rel')->nullable();
$table->string('wic_party_rel')->nullable();
$table->string('wic_amount')->nullable();
$table->string('wic_amount_type')->nullable();
$table->string('wic_amtbnk_name')->nullable();
$table->string('wic_amtbnk_cunm')->nullable();
$table->string('wic_fund_source')->nullable();
$table->string('wic_fund_use')->nullable();
$table->string('dr_cr_marker')->nullable();
$table->string('charge_code')->nullable();
$table->string('chrg_amt_local')->nullable();
$table->string('charge_category')->nullable();
$table->string('charge_account')->nullable();
$table->string('amount_fcy_1')->nullable();
$table->string('rate_1')->nullable();
$table->string('deal_rate')->nullable();
$table->string('l_wic_id')->nullable();
$table->string('account_1_co_code')->nullable();
$table->string('account_2_co_code')->nullable();
$table->string('l_charge_amt')->nullable();
$table->string('bl_cust_no')->nullable();
$table->string('stmt_no')->nullable();
$table->string('bil_customer')->nullable();
$table->string('value_date_2')->nullable();
$table->string('cheq_type')->nullable();
$table->string('cheque_number')->nullable();
$table->string('inputter')->nullable();
$table->string('authoriser')->nullable();
$table->string('bil_product')->nullable();
$table->string('fx_document')->nullable();
$table->string('fx_purpose')->nullable();
$table->text('narrative_2')->nullable();
$table->string('customer_2')->nullable();
$table->string('l_sms_1')->nullable();
$table->string('l_phone_1')->nullable();
$table->string('kyc_incom_rng')->nullable();
$table->string('wic_rt')->nullable();
$table->string('wic_rw')->nullable();
$table->string('ktp_kelurahan')->nullable();
$table->string('ktp_kecamatan')->nullable();
$table->string('ktp_provinsi')->nullable();
$table->string('wic_jenis_kelam')->nullable();
$table->string('kyc_sumber_dana')->nullable();
$table->string('l_manual_risk')->nullable();
$table->string('l_slip_no')->nullable();
$table->string('dest_bank_id')->nullable();
$table->text('dr_narrative')->nullable();
$table->string('inter_type')->nullable();
$table->string('l_va_number')->nullable();
$table->string('inter_bank_id')->nullable();
$table->text('term_narr')->nullable();
$table->string('currency_2')->nullable();
$table->string('amount_fcy_2')->nullable();
$table->string('rate_2')->nullable();
$table->string('customer_1')->nullable();
$table->string('last_version')->nullable();
$table->timestamps();
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::dropIfExists('tellers');
}
};