daengdeni/webstatement
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(GenerateClosingBalanceReportJob): Sederhanakan pendekatan eliminasi duplikasi seperti ExportStatementJob

Browse Source 
- Menghapus logika duplikasi yang kompleks dengan unique_hash dan pengecekan duplikasi
- Menyederhanakan proses delete data existing seperti pada ExportStatementJob
- Menghapus lock table dan verifikasi duplikasi yang rumit
- Menggunakan pendekatan langsung: hapus data lama -> proses ulang -> insert baru
- Menghapus field unique_hash yang tidak diperlukan lagi
- Menyederhanakan query builder tanpa eliminasi duplicate yang kompleks

Perubahan ini mengikuti pola yang sudah terbukti berhasil pada ExportStatementJob,
yang tidak memiliki masalah duplikasi dengan pendekatan yang lebih sederhana.
This commit is contained in:
Daeng Deni Mardaeni
2025-08-04 09:25:48 +07:00
parent 8736ccf5f8
commit 150d52f8da
1 changed files with 37 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

147
app/Jobs/GenerateClosingBalanceReportJob.php
View File

@@ -130,72 +130,40 @@
'group_name' => $this->groupName
];
// PERBAIKAN: Gunakan database transaction untuk memastikan atomicity
DB::beginTransaction();
try {
// PERBAIKAN: Lock table untuk mencegah race condition
DB::statement('LOCK TABLE processed_closing_balances IN EXCLUSIVE MODE');
// Delete existing processed data dengan verifikasi
$this->deleteExistingProcessedDataWithVerification($criteria);
// Sederhana: hapus data existing terlebih dahulu seperti ExportStatementJob
$this->deleteExistingProcessedData($criteria);
// Get opening balance
$runningBalance = $this->getOpeningBalance();
$sequenceNo = 0;
$totalProcessed = 0;
Log::info('Starting to process and save closing balance data with duplicate protection', [
Log::info('Starting to process closing balance data', [
'opening_balance' => $runningBalance,
'criteria' => $criteria
]);
// Build query
// Build query yang sederhana tanpa eliminasi duplicate rumit
$query = $this->buildTransactionQuery();
// PERBAIKAN: Collect all data first, then insert in single transaction
$allProcessedData = [];
$query->chunk($this->chunkSize, function ($transactions) use (&$runningBalance, &$sequenceNo, &$allProcessedData, &$totalProcessed) {
// Proses dan insert data langsung seperti ExportStatementJob
$query->chunk($this->chunkSize, function ($transactions) use (&$runningBalance, &$sequenceNo) {
$processedData = $this->prepareProcessedClosingBalanceData($transactions, $runningBalance, $sequenceNo);
if (!empty($processedData)) {
$allProcessedData = array_merge($allProcessedData, $processedData);
$totalProcessed += count($processedData);
DB::table('processed_closing_balances')->insert($processedData);
}
Log::info('Chunk processed and collected', [
'chunk_size' => count($processedData),
'total_collected' => count($allProcessedData),
'current_balance' => $runningBalance
]);
});
// PERBAIKAN: Insert all data in single operation
if (!empty($allProcessedData)) {
// Batch insert dengan chunk untuk menghindari memory limit
$insertChunks = array_chunk($allProcessedData, 1000);
foreach ($insertChunks as $chunk) {
// Menggunakan insertOrIgnore untuk mengabaikan duplikasi dan mencegah error unique constraint
DB::table('processed_closing_balances')->insertOrIgnore($chunk);
}
Log::info('All processed data inserted successfully', [
'total_records' => count($allProcessedData),
'insert_chunks' => count($insertChunks)
]);
}
// PERBAIKAN: Verify no duplicates after insert
$this->verifyNoDuplicatesAfterInsert($criteria);
DB::commit();
$recordCount = $this->getProcessedRecordCount();
Log::info('Closing balance data processing completed successfully', [
'final_sequence' => $sequenceNo,
'final_balance' => $runningBalance,
'total_processed' => $totalProcessed
'final_sequence' => $sequenceNo,
'final_balance' => $runningBalance,
'record_count' => $recordCount
]);
} catch (Exception $e) {
@@ -290,32 +258,17 @@
}
/**
* Build transaction query dengan eliminasi duplicate yang lebih ketat
* Membangun query transaksi dengan menghilangkan duplicate berdasarkan kombinasi lengkap
* Build transaction query dengan pendekatan sederhana tanpa eliminasi duplicate rumit
*/
private function buildTransactionQuery()
{
Log::info('Building transaction query with strict duplicate elimination', [
Log::info('Building transaction query', [
'group_name' => $this->groupName,
'account_number' => $this->accountNumber,
'period' => $this->period
]);
$modelClass = $this->getModelByGroup();
$tableName = (new $modelClass)->getTable();
// Gunakan kombinasi lengkap untuk eliminasi duplikasi
$uniqueIds = DB::table($tableName)
->select(DB::raw('MIN(id) as min_id'))
->where('account_number', $this->accountNumber)
->where('booking_date', $this->period)
->groupBy('trans_reference', 'amount_lcy', 'booking_date') // Kombinasi lengkap
->pluck('min_id');
Log::info('Unique transaction IDs identified', [
'total_unique_transactions' => $uniqueIds->count(),
'elimination_criteria' => 'trans_reference + amount_lcy + booking_date'
]);
$query = $modelClass::select([
'id',
@@ -324,21 +277,22 @@
'amount_lcy',
'date_time'
])
->with([
'ft' => function ($query) {
$query->select('ref_no', 'date_time', 'debit_acct_no', 'debit_value_date',
'credit_acct_no', 'bif_rcv_acct', 'bif_rcv_name', 'credit_value_date',
'at_unique_id', 'bif_ref_no', 'atm_order_id', 'recipt_no',
'api_iss_acct', 'api_benff_acct', 'authoriser', 'remarks',
'payment_details', 'ref_no', 'merchant_id', 'term_id');
},
'dc' => function ($query) {
$query->select('id', 'date_time');
}
])
->whereIn('id', $uniqueIds)
->orderBy('booking_date')
->orderBy('date_time');
->with([
'ft' => function ($query) {
$query->select('ref_no', 'date_time', 'debit_acct_no', 'debit_value_date',
'credit_acct_no', 'bif_rcv_acct', 'bif_rcv_name', 'credit_value_date',
'at_unique_id', 'bif_ref_no', 'atm_order_id', 'recipt_no',
'api_iss_acct', 'api_benff_acct', 'authoriser', 'remarks',
'payment_details', 'ref_no', 'merchant_id', 'term_id');
},
'dc' => function ($query) {
$query->select('id', 'date_time');
}
])
->where('account_number', $this->accountNumber)
->where('booking_date', $this->period)
->orderBy('booking_date')
->orderBy('date_time');
return $query;
}
@@ -364,8 +318,7 @@
}
/**
* Prepare processed closing balance data dengan validasi duplikasi yang ketat
* Mempersiapkan data closing balance dengan pencegahan duplikasi berbasis trans_reference unik
* Prepare processed closing balance data tanpa validasi duplikasi
*/
private function prepareProcessedClosingBalanceData($transactions, &$runningBalance, &$sequenceNo)
: array
@@ -373,23 +326,6 @@
$processedData = [];
foreach ($transactions as $transaction) {
// Validasi duplikasi berbasis trans_reference + amount_lcy + booking_date
$uniqueKey = md5($transaction->trans_reference . '_' . $transaction->amount_lcy);
$existingReferences = ProcessedClosingBalance::select('unique_hash')
->where('unique_hash', $uniqueKey)
->get();
// Periksa kombinasi trans_reference + amount_lcy, bukan hanya trans_reference
if ($existingReferences->count() > 0) {
Log::warning('Transaction already exists in database', [
'trans_reference' => $transaction->trans_reference,
'amount_lcy' => $transaction->amount_lcy
]);
continue;
}
$seenReferences[$uniqueKey] = true;
$sequenceNo++;
// Process transaction data
@@ -402,7 +338,7 @@
// Format transaction date
$transactionDate = $this->formatDateTime($processedTransactionData['date_time']);
// Prepare data untuk database insert dengan composite key yang lebih robust
// Prepare data untuk database insert tanpa unique_hash
$processedData[] = [
'account_number' => $this->accountNumber,
'period' => $this->period,
@@ -435,14 +371,11 @@
'closing_balance' => $runningBalance,
'created_at' => now(),
'updated_at' => now(),
// Tambahkan hash unik untuk memastikan keunikan
'unique_hash' => md5($transaction->trans_reference . '_' . $transaction->amount_lcy)
];
}
Log::info('Processed closing balance data prepared with duplicate prevention', [
'total_records' => count($processedData),
'skipped_duplicates' => count($transactions) - count($processedData)
Log::info('Processed closing balance data prepared', [
'total_records' => count($processedData)
]);
return $processedData;
@@ -650,8 +583,8 @@
->where('period', $this->period)
->where('group_name', $this->groupName)
->orderBy('sequence_no')
->chunk($this->chunkSize, function ($records) use ($filePath, &$sequenceCounter, &$processedHashes, &$csvContent) {
->chunk($this->chunkSize, function ($records) use ($filePath, &$sequenceCounter, &$processedHashes) {
$csvContent = [];
foreach ($records as $record) {
// Pengecekan unique_hash: skip jika sudah diproses
if (in_array($record->unique_hash, $processedHashes)) {
@@ -734,8 +667,7 @@
}
/**
* Delete existing processed data dengan logging
* Menghapus data processed yang sudah ada dengan logging untuk audit
* Delete existing processed data dengan pendekatan sederhana seperti ExportStatementJob
*/
private function deleteExistingProcessedData(array $criteria)
: void
@@ -745,12 +677,7 @@
$deletedCount = ProcessedClosingBalance::where('account_number', $criteria['account_number'])
->where('period', $criteria['period'])
->where('group_name', $criteria['group_name'])
->count();
ProcessedClosingBalance::where('account_number', $criteria['account_number'])
->where('period', $criteria['period'])
->where('group_name', $criteria['group_name'])
->delete();
->delete();
Log::info('Existing processed data deleted', [
'deleted_count' => $deletedCount,