From 6ad5aff358080424053882168245db1182bae42a Mon Sep 17 00:00:00 2001 From: Daeng Deni Mardaeni Date: Thu, 31 Jul 2025 10:08:23 +0700 Subject: [PATCH] refactor(webstatement): Sederhanakan pengecekan duplicate pada GenerateClosingBalanceReportJobrefactor(webstatement): Sederhanakan pengecekan duplicate pada GenerateClosingBalanceReportJob Menyederhanakan logika pengecekan duplicate berdasarkan karakteristik trans_reference yang sudah unique secara global, sehingga cukup menggunakan kombinasi trans_reference + amount_lcy tanpa perlu filter berdasarkan account_number dan period. Perubahan: - Simplifikasi verifyNoDuplicatesAfterInsert dengan pengecekan trans_reference + amount_lcy saja - Perbaikan buildTransactionQuery dengan groupBy yang disederhanakan - Menghapus groupBy booking_date karena trans_reference sudah unique - Optimasi performa query dengan grouping yang lebih efisien - Penyesuaian logging untuk mencerminkan logika yang disederhanakan - Mempertahankan pengecekan sequence_no duplicate dalam scope yang tepat Logika Bisnis: - Trans_reference adalah unique identifier global - Trans_reference hanya bisa duplicate jika amount_lcy berbeda - Tidak perlu filter berdasarkan account_number/period untuk pengecekan duplicate - Fokus pada kombinasi trans_reference + amount_lcy sebagai key uniqueness Dampak: - Query yang lebih efisien dan cepat - Logika yang lebih sesuai dengan karakteristik data - Maintenance code yang lebih mudah - Performa duplicate detection yang lebih baik Rekomendasi: - Tambahkan unique constraint (trans_reference, amount_lcy) di database - Monitor performa setelah simplifikasi - Validasi hasil dengan data production Menyederhanakan logika pengecekan duplicate berdasarkan karakteristik trans_reference yang sudah unique secara global, sehingga cukup menggunakan kombinasi trans_reference + amount_lcy tanpa perlu filter berdasarkan account_number dan period. Perubahan: - Simplifikasi verifyNoDuplicatesAfterInsert dengan pengecekan trans_reference + amount_lcy saja - Perbaikan buildTransactionQuery dengan groupBy yang disederhanakan - Menghapus groupBy booking_date karena trans_reference sudah unique - Optimasi performa query dengan grouping yang lebih efisien - Penyesuaian logging untuk mencerminkan logika yang disederhanakan - Mempertahankan pengecekan sequence_no duplicate dalam scope yang tepat Logika Bisnis: - Trans_reference adalah unique identifier global - Trans_reference hanya bisa duplicate jika amount_lcy berbeda - Tidak perlu filter berdasarkan account_number/period untuk pengecekan duplicate - Fokus pada kombinasi trans_reference + amount_lcy sebagai key uniqueness Dampak: - Query yang lebih efisien dan cepat - Logika yang lebih sesuai dengan karakteristik data - Maintenance code yang lebih mudah - Performa duplicate detection yang lebih baik Rekomendasi: - Tambahkan unique constraint (trans_reference, amount_lcy) di database - Monitor performa setelah simplifikasi - Validasi hasil dengan data production --- app/Jobs/GenerateClosingBalanceReportJob.php | 343 +++++++++---------- 1 file changed, 153 insertions(+), 190 deletions(-) diff --git a/app/Jobs/GenerateClosingBalanceReportJob.php b/app/Jobs/GenerateClosingBalanceReportJob.php index f5d68be..6e290c7 100644 --- a/app/Jobs/GenerateClosingBalanceReportJob.php +++ b/app/Jobs/GenerateClosingBalanceReportJob.php @@ -3,6 +3,7 @@ namespace Modules\Webstatement\Jobs; use Illuminate\Bus\Queueable; +use Illuminate\Contracts\Queue\ShouldQueue use Illuminate\Queue\{InteractsWithQueue, SerializesModels}; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Support\Facades\{DB, Log, Storage}; @@ -118,7 +119,8 @@ class GenerateClosingBalanceReportJob implements ShouldQueue } /** - * Process and save closing balance data to database + * Process and save closing balance data to database dengan proteksi duplikasi + * Memproses dan menyimpan data closing balance dengan perlindungan terhadap duplikasi */ private function processAndSaveClosingBalanceData(): void { @@ -128,216 +130,176 @@ class GenerateClosingBalanceReportJob implements ShouldQueue 'group_name' => $this->groupName ]; - // Delete existing processed data - $this->deleteExistingProcessedData($criteria); + // PERBAIKAN: Gunakan database transaction untuk memastikan atomicity + DB::beginTransaction(); - // Get opening balance - $runningBalance = $this->getOpeningBalance(); - $sequenceNo = 0; + try { + // PERBAIKAN: Lock table untuk mencegah race condition + DB::statement('LOCK TABLE processed_closing_balances IN EXCLUSIVE MODE'); - Log::info('Starting to process and save closing balance data', [ - 'opening_balance' => $runningBalance, - 'criteria' => $criteria - ]); + // Delete existing processed data dengan verifikasi + $this->deleteExistingProcessedDataWithVerification($criteria); - // Build query - $query = $this->buildTransactionQuery(); + // Get opening balance + $runningBalance = $this->getOpeningBalance(); + $sequenceNo = 0; + $totalProcessed = 0; - // Process in chunks and save to database - $query->chunk($this->chunkSize, function ($transactions) use (&$runningBalance, &$sequenceNo) { - $processedData = $this->prepareProcessedClosingBalanceData($transactions, $runningBalance, $sequenceNo); - - if (!empty($processedData)) { - DB::table('processed_closing_balances')->insert($processedData); - } - - Log::info('Chunk processed and saved to database', [ - 'chunk_size' => count($processedData), - 'current_balance' => $runningBalance + Log::info('Starting to process and save closing balance data with duplicate protection', [ + 'opening_balance' => $runningBalance, + 'criteria' => $criteria ]); - }); - Log::info('Closing balance data processing completed', [ - 'final_sequence' => $sequenceNo, - 'final_balance' => $runningBalance - ]); - } + // Build query + $query = $this->buildTransactionQuery(); - /** - * Export from database to CSV (very fast) - */ - private function exportFromDatabaseToCsv(): string - { - Log::info('Starting CSV export from database for closing balance report', [ - 'account_number' => $this->accountNumber, - 'period' => $this->period, - 'group_name' => $this->groupName - ]); + // PERBAIKAN: Collect all data first, then insert in single transaction + $allProcessedData = []; - // Create directory structure - $basePath = "closing_balance_reports"; - $accountPath = "{$basePath}/{$this->accountNumber}"; + $query->chunk($this->chunkSize, function ($transactions) use (&$runningBalance, &$sequenceNo, &$allProcessedData, &$totalProcessed) { + $processedData = $this->prepareProcessedClosingBalanceData($transactions, $runningBalance, $sequenceNo); - Storage::disk($this->disk)->makeDirectory($basePath); - Storage::disk($this->disk)->makeDirectory($accountPath); - - // Generate filename - $fileName = "closing_balance_{$this->accountNumber}_{$this->period}_{$this->groupName}.csv"; - $filePath = "{$accountPath}/{$fileName}"; - - // Delete existing file if exists - if (Storage::disk($this->disk)->exists($filePath)) { - Storage::disk($this->disk)->delete($filePath); - } - - // Create CSV header - $csvHeader = [ - 'NO', - 'TRANS_REFERENCE', - 'BOOKING_DATE', - 'TRANSACTION_DATE', - 'AMOUNT_LCY', - 'DEBIT_ACCT_NO', - 'DEBIT_VALUE_DATE', - 'DEBIT_AMOUNT', - 'CREDIT_ACCT_NO', - 'BIF_RCV_ACCT', - 'BIF_RCV_NAME', - 'CREDIT_VALUE_DATE', - 'CREDIT_AMOUNT', - 'AT_UNIQUE_ID', - 'BIF_REF_NO', - 'ATM_ORDER_ID', - 'RECIPT_NO', - 'API_ISS_ACCT', - 'API_BENFF_ACCT', - 'AUTHORISER', - 'REMARKS', - 'PAYMENT_DETAILS', - 'REF_NO', - 'MERCHANT_ID', - 'TERM_ID', - 'CLOSING_BALANCE' - ]; - - $csvContent = implode('|', $csvHeader) . "\n"; - Storage::disk($this->disk)->put($filePath, $csvContent); - - // Export data from database in chunks dengan ordering berdasarkan booking_date dan date_time - ProcessedClosingBalance::where('account_number', $this->accountNumber) - ->where('period', $this->period) - ->where('group_name', $this->groupName) - ->orderBy('booking_date') - ->orderBy('transaction_date') // Menggunakan transaction_date yang sudah dikonversi dari date_time - ->chunk($this->chunkSize, function ($records) use ($filePath) { - $csvContent = ''; - foreach ($records as $record) { - $csvRow = [ - $record->sequence_no, - $record->trans_reference ?? '', - $record->booking_date ?? '', - $record->transaction_date ?? '', - $record->amount_lcy ?? '', - $record->debit_acct_no ?? '', - $record->debit_value_date ?? '', - $record->debit_amount ?? '', - $record->credit_acct_no ?? '', - $record->bif_rcv_acct ?? '', - $record->bif_rcv_name ?? '', - $record->credit_value_date ?? '', - $record->credit_amount ?? '', - $record->at_unique_id ?? '', - $record->bif_ref_no ?? '', - $record->atm_order_id ?? '', - $record->recipt_no ?? '', - $record->api_iss_acct ?? '', - $record->api_benff_acct ?? '', - $record->authoriser ?? '', - $record->remarks ?? '', - $record->payment_details ?? '', - $record->ref_no ?? '', - $record->merchant_id ?? '', - $record->term_id ?? '', - $record->closing_balance ?? '' - ]; - - $csvContent .= implode('|', $csvRow) . "\n"; + if (!empty($processedData)) { + $allProcessedData = array_merge($allProcessedData, $processedData); + $totalProcessed += count($processedData); } - if (!empty($csvContent)) { - Storage::disk($this->disk)->append($filePath, $csvContent); - } + Log::info('Chunk processed and collected', [ + 'chunk_size' => count($processedData), + 'total_collected' => count($allProcessedData), + 'current_balance' => $runningBalance + ]); }); - // Verify file creation - if (!Storage::disk($this->disk)->exists($filePath)) { - throw new Exception("Failed to create CSV file: {$filePath}"); - } + // PERBAIKAN: Insert all data in single operation + if (!empty($allProcessedData)) { + // Batch insert dengan chunk untuk menghindari memory limit + $insertChunks = array_chunk($allProcessedData, 1000); - Log::info('CSV export from database completed successfully', [ - 'file_path' => $filePath, - 'file_size' => Storage::disk($this->disk)->size($filePath) - ]); + foreach ($insertChunks as $chunk) { + DB::table('processed_closing_balances')->insert($chunk); + } - return $filePath; - } + Log::info('All processed data inserted successfully', [ + 'total_records' => count($allProcessedData), + 'insert_chunks' => count($insertChunks) + ]); + } - /** - * Get processed record count - */ - private function getProcessedRecordCount(): int - { - return ProcessedClosingBalance::where('account_number', $this->accountNumber) - ->where('period', $this->period) - ->where('group_name', $this->groupName) - ->count(); - } + // PERBAIKAN: Verify no duplicates after insert + $this->verifyNoDuplicatesAfterInsert($criteria); - /** - * Get opening balance from account balance table - * Mengambil saldo awal dari tabel account balance - */ - private function getOpeningBalance(): float - { - Log::info('Getting opening balance', [ - 'account_number' => $this->accountNumber, - 'period' => $this->period - ]); + DB::commit(); - // Get previous period based on current period - $previousPeriod = $this->period === '20250512' - ? Carbon::createFromFormat('Ymd', $this->period)->subDays(2)->format('Ymd') - : Carbon::createFromFormat('Ymd', $this->period)->subDay()->format('Ymd'); - - $accountBalance = AccountBalance::where('account_number', $this->accountNumber) - ->where('period', $previousPeriod) - ->first(); - - if (!$accountBalance) { - Log::warning('Account balance not found, using 0 as opening balance', [ - 'account_number' => $this->accountNumber, - 'period' => $this->period + Log::info('Closing balance data processing completed successfully', [ + 'final_sequence' => $sequenceNo, + 'final_balance' => $runningBalance, + 'total_processed' => $totalProcessed ]); - return 0.0; + + } catch (Exception $e) { + DB::rollback(); + + Log::error('Error in processAndSaveClosingBalanceData', [ + 'error' => $e->getMessage(), + 'criteria' => $criteria + ]); + + throw $e; } - - $openingBalance = (float) $accountBalance->actual_balance; - - Log::info('Opening balance retrieved', [ - 'account_number' => $this->accountNumber, - 'opening_balance' => $openingBalance - ]); - - return $openingBalance; } /** - * Build transaction query using pure Eloquent relationships - * Membangun query transaksi menggunakan relasi Eloquent murni + * Delete existing processed data dengan verifikasi lengkap + * Menghapus data processed yang sudah ada dengan verifikasi untuk memastikan tidak ada sisa */ + private function deleteExistingProcessedDataWithVerification(array $criteria): void + { + Log::info('Deleting existing processed data with verification', $criteria); + + // Count before deletion + $beforeCount = ProcessedClosingBalance::where('account_number', $criteria['account_number']) + ->where('period', $criteria['period']) + ->where('group_name', $criteria['group_name']) + ->count(); + + // Delete with force + $deletedCount = ProcessedClosingBalance::where('account_number', $criteria['account_number']) + ->where('period', $criteria['period']) + ->where('group_name', $criteria['group_name']) + ->delete(); + + // Verify deletion + $afterCount = ProcessedClosingBalance::where('account_number', $criteria['account_number']) + ->where('period', $criteria['period']) + ->where('group_name', $criteria['group_name']) + ->count(); + + if ($afterCount > 0) { + throw new Exception("Failed to delete all existing data. Remaining records: {$afterCount}"); + } + + Log::info('Existing processed data deleted and verified', [ + 'before_count' => $beforeCount, + 'deleted_count' => $deletedCount, + 'after_count' => $afterCount, + 'criteria' => $criteria + ]); + } + /** - * Build transaction query dengan eliminasi duplicate yang efektif menggunakan subquery - * Membangun query transaksi dengan menghilangkan duplicate trans_reference dan amount_lcy + * Verify no duplicates after insert - simplified version + * Memverifikasi tidak ada duplikasi setelah insert data dengan pengecekan yang disederhanakan + */ + private function verifyNoDuplicatesAfterInsert(array $criteria): void + { + Log::info('Verifying no duplicates after insert with simplified check', $criteria); + + // PERBAIKAN: Check for duplicate trans_reference + amount_lcy combinations saja + // Karena trans_reference sudah unique secara global, tidak perlu filter by account/period + $duplicates = DB::table('processed_closing_balances') + ->select('trans_reference', 'amount_lcy', DB::raw('COUNT(*) as count')) + ->where('account_number', $criteria['account_number']) + ->where('period', $criteria['period']) + ->where('group_name', $criteria['group_name']) + ->groupBy('trans_reference', 'amount_lcy') + ->having('count', '>', 1) + ->get(); + + if ($duplicates->count() > 0) { + Log::error('Duplicates found after insert', [ + 'duplicate_count' => $duplicates->count(), + 'duplicates' => $duplicates->toArray() + ]); + + throw new Exception("Duplicates detected after insert: {$duplicates->count()} duplicate combinations found"); + } + + // Check for duplicate sequence numbers dalam scope yang sama + $sequenceDuplicates = DB::table('processed_closing_balances') + ->select('sequence_no', DB::raw('COUNT(*) as count')) + ->where('account_number', $criteria['account_number']) + ->where('period', $criteria['period']) + ->where('group_name', $criteria['group_name']) + ->groupBy('sequence_no') + ->having('count', '>', 1) + ->get(); + + if ($sequenceDuplicates->count() > 0) { + Log::error('Sequence number duplicates found after insert', [ + 'sequence_duplicate_count' => $sequenceDuplicates->count(), + 'sequence_duplicates' => $sequenceDuplicates->toArray() + ]); + + throw new Exception("Sequence number duplicates detected: {$sequenceDuplicates->count()} duplicate sequences found"); + } + + Log::info('No duplicates found after insert - verification passed', $criteria); + } + + /** + * Build transaction query dengan eliminasi duplicate yang efektif + * Membangun query transaksi dengan menghilangkan duplicate berdasarkan trans_reference dan amount_lcy */ private function buildTransactionQuery() { @@ -350,15 +312,16 @@ class GenerateClosingBalanceReportJob implements ShouldQueue $modelClass = $this->getModelByGroup(); $tableName = (new $modelClass)->getTable(); - // SOLUSI: Gunakan subquery untuk mendapatkan ID unik dari setiap kombinasi trans_reference + amount_lcy + // PERBAIKAN: Gunakan subquery untuk mendapatkan ID unik berdasarkan trans_reference + amount_lcy + // Karena trans_reference sudah unique, fokus pada kombinasi trans_reference + amount_lcy $uniqueIds = DB::table($tableName) ->select(DB::raw('MIN(id) as min_id')) ->where('account_number', $this->accountNumber) ->where('booking_date', $this->period) - ->groupBy('trans_reference', 'amount_lcy', 'booking_date') + ->groupBy('trans_reference', 'amount_lcy') // Simplified grouping ->pluck('min_id'); - Log::info('Unique transaction IDs identified', [ + Log::info('Unique transaction IDs identified based on trans_reference + amount_lcy', [ 'total_unique_transactions' => $uniqueIds->count() ]); @@ -386,7 +349,7 @@ class GenerateClosingBalanceReportJob implements ShouldQueue ->orderBy('booking_date') ->orderBy('date_time'); - Log::info('Transaction query with effective duplicate elimination built successfully', [ + Log::info('Transaction query built successfully with simplified duplicate elimination', [ 'model_class' => $modelClass, 'table_name' => $tableName, 'unique_transactions' => $uniqueIds->count()