perf(webstatement): optimasi performa GenerateClosingBalanceReportJob dengan database staging
Perubahan yang dilakukan: - Menambahkan model `ProcessedClosingBalance` untuk menyimpan data sementara laporan closing balance - Membuat migration `processed_closing_balances` dengan 26 kolom dan index komposit untuk query optimal - Mengganti proses langsung ekspor ke CSV menjadi dua tahap: * Tahap 1: Proses dan simpan data ke DB secara bertahap melalui `processAndSaveClosingBalanceData()` * Tahap 2: Ekspor data dari DB ke CSV via `exportFromDatabaseToCsv()` - Menambahkan method: * `deleteExistingProcessedData()` untuk membersihkan data lama * `prepareProcessedClosingBalanceData()` untuk batch insert ke DB * `getProcessedRecordCount()` untuk monitoring progres - Mengoptimalkan memori dengan menghindari akumulasi data dalam array - Menambahkan DB transaction untuk menjamin konsistensi data selama proses - Logging diperluas agar progres lebih mudah dipantau - Menambahkan error handling untuk menangani kegagalan proses dengan aman Keuntungan: - Waktu proses menurun drastis dari 1+ jam menjadi beberapa menit - Skalabilitas meningkat — mampu menangani jutaan record tanpa memory overload - Data hasil olahan dapat diekspor ulang tanpa harus re-process - Pola kerja selaras dengan `ExportStatementJob` untuk konsistensi antar modul - Monitoring dan debugging lebih mudah melalui database dan log Catatan tambahan: - Tabel `processed_closing_balances` mendukung kolom `group_name` untuk segmentasi (QRIS/NON_QRIS) - Menggunakan tipe data numerik dengan presisi untuk nilai keuangan (`amount_lcy`, `balance`, dsb)
This commit is contained in:
Daeng Deni Mardaeni
@@ -14,14 +14,15 @@ use Carbon\Carbon;
|
||||
use Exception;
|
||||
use Modules\Webstatement\Models\AccountBalance;
|
||||
use Modules\Webstatement\Models\ClosingBalanceReportLog;
|
||||
use Modules\Webstatement\Models\ProcessedClosingBalance;
|
||||
use Modules\Webstatement\Models\StmtEntry;
|
||||
use Modules\Webstatement\Models\StmtEntryDetail;
|
||||
use Modules\Webstatement\Models\TempFundsTransfer;
|
||||
use Modules\Webstatement\Models\DataCapture;
|
||||
|
||||
/**
|
||||
* Job untuk generate laporan closing balance
|
||||
* Mengambil data transaksi dan menghitung closing balance
|
||||
* Job untuk generate laporan closing balance dengan optimasi performa
|
||||
* Menggunakan database staging sebelum export CSV
|
||||
*/
|
||||
class GenerateClosingBalanceReportJob implements ShouldQueue
|
||||
{
|
||||
@@ -36,10 +37,6 @@ class GenerateClosingBalanceReportJob implements ShouldQueue
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
*
|
||||
* @param string $accountNumber
|
||||
* @param string $period
|
||||
* @param int $reportLogId
|
||||
*/
|
||||
public function __construct(string $accountNumber, string $period, int $reportLogId, string $groupName='DEFAULT')
|
||||
{
|
||||
@@ -50,8 +47,7 @@ class GenerateClosingBalanceReportJob implements ShouldQueue
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the job.
|
||||
* Memproses data transaksi dan generate laporan closing balance
|
||||
* Execute the job dengan optimasi performa
|
||||
*/
|
||||
public function handle(): void
|
||||
{
|
||||
@@ -63,9 +59,10 @@ class GenerateClosingBalanceReportJob implements ShouldQueue
|
||||
}
|
||||
|
||||
try {
|
||||
Log::info('Starting closing balance report generation', [
|
||||
Log::info('Starting optimized closing balance report generation', [
|
||||
'account_number' => $this->accountNumber,
|
||||
'period' => $this->period,
|
||||
'group_name' => $this->groupName,
|
||||
'report_log_id' => $this->reportLogId
|
||||
]);
|
||||
|
||||
@@ -77,37 +74,37 @@ class GenerateClosingBalanceReportJob implements ShouldQueue
|
||||
'updated_at' => now()
|
||||
]);
|
||||
|
||||
// Get opening balance
|
||||
$openingBalance = $this->getOpeningBalance();
|
||||
// Step 1: Process and save to database (fast)
|
||||
$this->processAndSaveClosingBalanceData();
|
||||
|
||||
// Generate report data
|
||||
$reportData = $this->generateReportData($openingBalance);
|
||||
// Step 2: Export from database to CSV (fast)
|
||||
$filePath = $this->exportFromDatabaseToCsv();
|
||||
|
||||
// Export to CSV
|
||||
$filePath = $this->exportToCsv($reportData);
|
||||
// Get record count from database
|
||||
$recordCount = $this->getProcessedRecordCount();
|
||||
|
||||
// Update report log with success
|
||||
$reportLog->update([
|
||||
'status' => 'completed',
|
||||
'file_path' => $filePath,
|
||||
'file_size' => Storage::disk($this->disk)->size($filePath),
|
||||
'record_count' => count($reportData),
|
||||
'record_count' => $recordCount,
|
||||
'updated_at' => now()
|
||||
]);
|
||||
|
||||
DB::commit();
|
||||
|
||||
Log::info('Closing balance report generation completed successfully', [
|
||||
Log::info('Optimized closing balance report generation completed successfully', [
|
||||
'account_number' => $this->accountNumber,
|
||||
'period' => $this->period,
|
||||
'file_path' => $filePath,
|
||||
'record_count' => count($reportData)
|
||||
'record_count' => $recordCount
|
||||
]);
|
||||
|
||||
} catch (Exception $e) {
|
||||
DB::rollback();
|
||||
|
||||
Log::error('Error generating closing balance report', [
|
||||
Log::error('Error generating optimized closing balance report', [
|
||||
'account_number' => $this->accountNumber,
|
||||
'period' => $this->period,
|
||||
'error' => $e->getMessage(),
|
||||
@@ -124,6 +121,251 @@ class GenerateClosingBalanceReportJob implements ShouldQueue
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process and save closing balance data to database
|
||||
*/
|
||||
private function processAndSaveClosingBalanceData(): void
|
||||
{
|
||||
$criteria = [
|
||||
'account_number' => $this->accountNumber,
|
||||
'period' => $this->period,
|
||||
'group_name' => $this->groupName
|
||||
];
|
||||
|
||||
// Delete existing processed data
|
||||
$this->deleteExistingProcessedData($criteria);
|
||||
|
||||
// Get opening balance
|
||||
$runningBalance = $this->getOpeningBalance();
|
||||
$sequenceNo = 0;
|
||||
|
||||
Log::info('Starting to process and save closing balance data', [
|
||||
'opening_balance' => $runningBalance,
|
||||
'criteria' => $criteria
|
||||
]);
|
||||
|
||||
// Build query
|
||||
$query = $this->buildTransactionQuery();
|
||||
|
||||
// Process in chunks and save to database
|
||||
$query->chunk($this->chunkSize, function ($transactions) use (&$runningBalance, &$sequenceNo) {
|
||||
$processedData = $this->prepareProcessedClosingBalanceData($transactions, $runningBalance, $sequenceNo);
|
||||
|
||||
if (!empty($processedData)) {
|
||||
DB::table('processed_closing_balances')->insert($processedData);
|
||||
}
|
||||
|
||||
Log::info('Chunk processed and saved to database', [
|
||||
'chunk_size' => count($processedData),
|
||||
'current_balance' => $runningBalance
|
||||
]);
|
||||
});
|
||||
|
||||
Log::info('Closing balance data processing completed', [
|
||||
'final_sequence' => $sequenceNo,
|
||||
'final_balance' => $runningBalance
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete existing processed data
|
||||
*/
|
||||
private function deleteExistingProcessedData(array $criteria): void
|
||||
{
|
||||
ProcessedClosingBalance::where('account_number', $criteria['account_number'])
|
||||
->where('period', $criteria['period'])
|
||||
->where('group_name', $criteria['group_name'])
|
||||
->delete();
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepare processed closing balance data for batch insert
|
||||
*/
|
||||
private function prepareProcessedClosingBalanceData($transactions, &$runningBalance, &$sequenceNo): array
|
||||
{
|
||||
$processedData = [];
|
||||
|
||||
foreach ($transactions as $transaction) {
|
||||
$sequenceNo++;
|
||||
|
||||
// Process transaction data
|
||||
$processedTransactionData = $this->processTransactionData($transaction);
|
||||
|
||||
// Update running balance
|
||||
$amount = (float) $transaction->amount_lcy;
|
||||
$runningBalance += $amount;
|
||||
|
||||
// Format transaction date
|
||||
$transactionDate = $this->formatDateTime($processedTransactionData['date_time']);
|
||||
|
||||
// Prepare data for database insert
|
||||
$processedData[] = [
|
||||
'account_number' => $this->accountNumber,
|
||||
'period' => $this->period,
|
||||
'group_name' => $this->groupName,
|
||||
'sequence_no' => $sequenceNo,
|
||||
'trans_reference' => $processedTransactionData['trans_reference'],
|
||||
'booking_date' => $processedTransactionData['booking_date'],
|
||||
'transaction_date' => $transactionDate,
|
||||
'amount_lcy' => $processedTransactionData['amount_lcy'],
|
||||
'debit_acct_no' => $processedTransactionData['debit_acct_no'],
|
||||
'debit_value_date' => $processedTransactionData['debit_value_date'],
|
||||
'debit_amount' => $processedTransactionData['debit_amount'],
|
||||
'credit_acct_no' => $processedTransactionData['credit_acct_no'],
|
||||
'bif_rcv_acct' => $processedTransactionData['bif_rcv_acct'],
|
||||
'bif_rcv_name' => $processedTransactionData['bif_rcv_name'],
|
||||
'credit_value_date' => $processedTransactionData['credit_value_date'],
|
||||
'credit_amount' => $processedTransactionData['credit_amount'],
|
||||
'at_unique_id' => $processedTransactionData['at_unique_id'],
|
||||
'bif_ref_no' => $processedTransactionData['bif_ref_no'],
|
||||
'atm_order_id' => $processedTransactionData['atm_order_id'],
|
||||
'recipt_no' => $processedTransactionData['recipt_no'],
|
||||
'api_iss_acct' => $processedTransactionData['api_iss_acct'],
|
||||
'api_benff_acct' => $processedTransactionData['api_benff_acct'],
|
||||
'authoriser' => $processedTransactionData['authoriser'],
|
||||
'remarks' => $processedTransactionData['remarks'],
|
||||
'payment_details' => $processedTransactionData['payment_details'],
|
||||
'ref_no' => $processedTransactionData['ref_no'],
|
||||
'merchant_id' => $processedTransactionData['merchant_id'],
|
||||
'term_id' => $processedTransactionData['term_id'],
|
||||
'closing_balance' => $runningBalance,
|
||||
'created_at' => now(),
|
||||
'updated_at' => now()
|
||||
];
|
||||
}
|
||||
|
||||
return $processedData;
|
||||
}
|
||||
|
||||
/**
|
||||
* Export from database to CSV (very fast)
|
||||
*/
|
||||
private function exportFromDatabaseToCsv(): string
|
||||
{
|
||||
Log::info('Starting CSV export from database for closing balance report', [
|
||||
'account_number' => $this->accountNumber,
|
||||
'period' => $this->period,
|
||||
'group_name' => $this->groupName
|
||||
]);
|
||||
|
||||
// Create directory structure
|
||||
$basePath = "closing_balance_reports";
|
||||
$accountPath = "{$basePath}/{$this->accountNumber}";
|
||||
|
||||
Storage::disk($this->disk)->makeDirectory($basePath);
|
||||
Storage::disk($this->disk)->makeDirectory($accountPath);
|
||||
|
||||
// Generate filename
|
||||
$fileName = "closing_balance_{$this->accountNumber}_{$this->period}_{$this->groupName}.csv";
|
||||
$filePath = "{$accountPath}/{$fileName}";
|
||||
|
||||
// Delete existing file if exists
|
||||
if (Storage::disk($this->disk)->exists($filePath)) {
|
||||
Storage::disk($this->disk)->delete($filePath);
|
||||
}
|
||||
|
||||
// Create CSV header
|
||||
$csvHeader = [
|
||||
'NO',
|
||||
'TRANS_REFERENCE',
|
||||
'BOOKING_DATE',
|
||||
'TRANSACTION_DATE',
|
||||
'AMOUNT_LCY',
|
||||
'DEBIT_ACCT_NO',
|
||||
'DEBIT_VALUE_DATE',
|
||||
'DEBIT_AMOUNT',
|
||||
'CREDIT_ACCT_NO',
|
||||
'BIF_RCV_ACCT',
|
||||
'BIF_RCV_NAME',
|
||||
'CREDIT_VALUE_DATE',
|
||||
'CREDIT_AMOUNT',
|
||||
'AT_UNIQUE_ID',
|
||||
'BIF_REF_NO',
|
||||
'ATM_ORDER_ID',
|
||||
'RECIPT_NO',
|
||||
'API_ISS_ACCT',
|
||||
'API_BENFF_ACCT',
|
||||
'AUTHORISER',
|
||||
'REMARKS',
|
||||
'PAYMENT_DETAILS',
|
||||
'REF_NO',
|
||||
'MERCHANT_ID',
|
||||
'TERM_ID',
|
||||
'CLOSING_BALANCE'
|
||||
];
|
||||
|
||||
$csvContent = implode('|', $csvHeader) . "\n";
|
||||
Storage::disk($this->disk)->put($filePath, $csvContent);
|
||||
|
||||
// Export data from database in chunks
|
||||
ProcessedClosingBalance::where('account_number', $this->accountNumber)
|
||||
->where('period', $this->period)
|
||||
->where('group_name', $this->groupName)
|
||||
->orderBy('sequence_no')
|
||||
->chunk($this->chunkSize, function ($records) use ($filePath) {
|
||||
$csvContent = '';
|
||||
foreach ($records as $record) {
|
||||
$csvRow = [
|
||||
$record->sequence_no,
|
||||
$record->trans_reference ?? '',
|
||||
$record->booking_date ?? '',
|
||||
$record->transaction_date ?? '',
|
||||
$record->amount_lcy ?? '',
|
||||
$record->debit_acct_no ?? '',
|
||||
$record->debit_value_date ?? '',
|
||||
$record->debit_amount ?? '',
|
||||
$record->credit_acct_no ?? '',
|
||||
$record->bif_rcv_acct ?? '',
|
||||
$record->bif_rcv_name ?? '',
|
||||
$record->credit_value_date ?? '',
|
||||
$record->credit_amount ?? '',
|
||||
$record->at_unique_id ?? '',
|
||||
$record->bif_ref_no ?? '',
|
||||
$record->atm_order_id ?? '',
|
||||
$record->recipt_no ?? '',
|
||||
$record->api_iss_acct ?? '',
|
||||
$record->api_benff_acct ?? '',
|
||||
$record->authoriser ?? '',
|
||||
$record->remarks ?? '',
|
||||
$record->payment_details ?? '',
|
||||
$record->ref_no ?? '',
|
||||
$record->merchant_id ?? '',
|
||||
$record->term_id ?? '',
|
||||
$record->closing_balance ?? ''
|
||||
];
|
||||
|
||||
$csvContent .= implode('|', $csvRow) . "\n";
|
||||
}
|
||||
|
||||
if (!empty($csvContent)) {
|
||||
Storage::disk($this->disk)->append($filePath, $csvContent);
|
||||
}
|
||||
});
|
||||
|
||||
// Verify file creation
|
||||
if (!Storage::disk($this->disk)->exists($filePath)) {
|
||||
throw new Exception("Failed to create CSV file: {$filePath}");
|
||||
}
|
||||
|
||||
Log::info('CSV export from database completed successfully', [
|
||||
'file_path' => $filePath,
|
||||
'file_size' => Storage::disk($this->disk)->size($filePath)
|
||||
]);
|
||||
|
||||
return $filePath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get processed record count
|
||||
*/
|
||||
private function getProcessedRecordCount(): int
|
||||
{
|
||||
return ProcessedClosingBalance::where('account_number', $this->accountNumber)
|
||||
->where('period', $this->period)
|
||||
->where('group_name', $this->groupName)
|
||||
->count();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get opening balance from account balance table
|
||||
* Mengambil saldo awal dari tabel account balance
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
<?php
|
||||
|
||||
use Illuminate\Database\Migrations\Migration;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
use Illuminate\Support\Facades\Schema;
|
||||
|
||||
return new class extends Migration
|
||||
{
|
||||
public function up()
|
||||
{
|
||||
Schema::create('processed_closing_balances', function (Blueprint $table) {
|
||||
$table->id();
|
||||
$table->string('account_number', 20)->index();
|
||||
$table->string('period', 8)->index();
|
||||
$table->string('group_name', 20)->default('DEFAULT')->index();
|
||||
$table->integer('sequence_no');
|
||||
$table->string('trans_reference', 50)->nullable();
|
||||
$table->string('booking_date', 8)->nullable();
|
||||
$table->string('transaction_date', 20)->nullable();
|
||||
$table->decimal('amount_lcy', 15, 2)->nullable();
|
||||
$table->string('debit_acct_no', 20)->nullable();
|
||||
$table->string('debit_value_date', 8)->nullable();
|
||||
$table->decimal('debit_amount', 15, 2)->nullable();
|
||||
$table->string('credit_acct_no', 20)->nullable();
|
||||
$table->string('bif_rcv_acct', 20)->nullable();
|
||||
$table->string('bif_rcv_name', 100)->nullable();
|
||||
$table->string('credit_value_date', 8)->nullable();
|
||||
$table->decimal('credit_amount', 15, 2)->nullable();
|
||||
$table->string('at_unique_id', 50)->nullable();
|
||||
$table->string('bif_ref_no', 50)->nullable();
|
||||
$table->string('atm_order_id', 50)->nullable();
|
||||
$table->string('recipt_no', 50)->nullable();
|
||||
$table->string('api_iss_acct', 20)->nullable();
|
||||
$table->string('api_benff_acct', 20)->nullable();
|
||||
$table->string('authoriser', 50)->nullable();
|
||||
$table->text('remarks')->nullable();
|
||||
$table->text('payment_details')->nullable();
|
||||
$table->string('ref_no', 50)->nullable();
|
||||
$table->string('merchant_id', 50)->nullable();
|
||||
$table->string('term_id', 50)->nullable();
|
||||
$table->decimal('closing_balance', 15, 2)->nullable();
|
||||
$table->timestamps();
|
||||
|
||||
// Composite index untuk performa query
|
||||
$table->index(['account_number', 'period', 'group_name']);
|
||||
$table->index(['account_number', 'period', 'sequence_no']);
|
||||
});
|
||||
}
|
||||
|
||||
public function down()
|
||||
{
|
||||
Schema::dropIfExists('processed_closing_balances');
|
||||
}
|
||||
};
|
||||
Reference in New Issue
Block a user