daengdeni/webstatement
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

🔧 fix(job): perbaiki transaksi bersarang & optimasi batch processing di GenerateClosingBalanceReportJob

Browse Source 
- Hapus transaksi bersarang:
  - Pindahkan DB::beginTransaction() & DB::commit() dari processAndSaveClosingBalanceData() ke handle() menggunakan DB::transaction()
  - Cegah error max_lock_per_transaction pada PostgreSQL

- Implementasi batch processing:
  - Tambah metode batchUpdateOrCreate()
  - Ambil data existing dalam satu query via whereIn()
  - Pisahkan data menjadi toInsert & toUpdate
  - Gunakan DB::table()->insert() untuk batch insert
  - Lakukan update individual untuk data yang sudah ada

- Penyederhanaan error handling:
  - Hapus try-catch di processAndSaveClosingBalanceData() karena sudah ditangani di handle()

- Penambahan logging:
  - Tambah log informasi untuk monitoring batch insert/update

- Optimasi performa:
  - Kurangi beban database & mencegah duplikasi data
  - Gunakan pendekatan "delete-first, then insert" seperti ExportStatementJob
This commit is contained in:
Daeng Deni Mardaeni
2025-08-05 13:49:58 +07:00
parent aae0c4ab15
commit 8a6469ecc9
1 changed files with 115 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

193
app/Jobs/GenerateClosingBalanceReportJob.php
View File

@@ -44,8 +44,7 @@
/** /**
* Execute the job dengan optimasi performa * Execute the job dengan optimasi performa
*/ */
public function handle() public function handle(): void
: void
{ {
$reportLog = ClosingBalanceReportLog::find($this->reportLogId); $reportLog = ClosingBalanceReportLog::find($this->reportLogId);
@@ -62,44 +61,41 @@
'report_log_id' => $this->reportLogId 'report_log_id' => $this->reportLogId
]); ]);
DB::beginTransaction();
// Update status to processing // Update status to processing
$reportLog->update([ $reportLog->update([
'status' => 'processing', 'status' => 'processing',
'updated_at' => now() 'updated_at' => now()
]); ]);
// Step 1: Process and save to database (fast) // Gunakan satu transaksi untuk seluruh proses
$this->processAndSaveClosingBalanceData(); DB::transaction(function () use ($reportLog) {
// Step 1: Process and save to database (fast)
$this->processAndSaveClosingBalanceData();
// Step 2: Export from database to CSV (fast) // Step 2: Export from database to CSV (fast)
$filePath = $this->exportFromDatabaseToCsv(); $filePath = $this->exportFromDatabaseToCsv();
// Get record count from database // Get record count from database
$recordCount = $this->getProcessedRecordCount(); $recordCount = $this->getProcessedRecordCount();
// Update report log with success // Update report log with success
$reportLog->update([ $reportLog->update([
'status' => 'completed', 'status' => 'completed',
'file_path' => $filePath, 'file_path' => $filePath,
'file_size' => Storage::disk($this->disk)->size($filePath), 'file_size' => Storage::disk($this->disk)->size($filePath),
'record_count' => $recordCount, 'record_count' => $recordCount,
'updated_at' => now() 'updated_at' => now()
]); ]);
DB::commit(); Log::info('Optimized closing balance report generation completed successfully', [
'account_number' => $this->accountNumber,
Log::info('Optimized closing balance report generation completed successfully', [ 'period' => $this->period,
'account_number' => $this->accountNumber, 'file_path' => $filePath,
'period' => $this->period, 'record_count' => $recordCount
'file_path' => $filePath, ]);
'record_count' => $recordCount });
]);
} catch (Exception $e) { } catch (Exception $e) {
DB::rollback();
Log::error('Error generating optimized closing balance report', [ Log::error('Error generating optimized closing balance report', [
'account_number' => $this->accountNumber, 'account_number' => $this->accountNumber,
'period' => $this->period, 'period' => $this->period,
@@ -121,8 +117,7 @@
* Process and save closing balance data to database dengan proteksi duplikasi * Process and save closing balance data to database dengan proteksi duplikasi
* Memproses dan menyimpan data closing balance dengan perlindungan terhadap duplikasi * Memproses dan menyimpan data closing balance dengan perlindungan terhadap duplikasi
*/ */
private function processAndSaveClosingBalanceData() private function processAndSaveClosingBalanceData(): void
: void
{ {
$criteria = [ $criteria = [
'account_number' => $this->accountNumber, 'account_number' => $this->accountNumber,
@@ -130,62 +125,41 @@
'group_name' => $this->groupName 'group_name' => $this->groupName
]; ];
DB::beginTransaction(); // HAPUS DB::beginTransaction() - sudah ada di handle()
try { // Sederhana: hapus data existing terlebih dahulu seperti ExportStatementJob
// Sederhana: hapus data existing terlebih dahulu seperti ExportStatementJob $this->deleteExistingProcessedData($criteria);
$this->deleteExistingProcessedData($criteria);
// Get opening balance // Get opening balance
$runningBalance = $this->getOpeningBalance(); $runningBalance = $this->getOpeningBalance();
$sequenceNo = 0; $sequenceNo = 0;
Log::info('Starting to process closing balance data', [ Log::info('Starting to process closing balance data', [
'opening_balance' => $runningBalance, 'opening_balance' => $runningBalance,
'criteria' => $criteria 'criteria' => $criteria
]); ]);
// Build query yang sederhana tanpa eliminasi duplicate rumit // Build query yang sederhana tanpa eliminasi duplicate rumit
$query = $this->buildTransactionQuery(); $query = $this->buildTransactionQuery();
// Proses dan insert data langsung seperti ExportStatementJob // Proses dan insert data dengan batch updateOrCreate untuk efisiensi
$query->chunk($this->chunkSize, function ($transactions) use (&$runningBalance, &$sequenceNo) { $query->chunk($this->chunkSize, function ($transactions) use (&$runningBalance, &$sequenceNo) {
$processedData = $this->prepareProcessedClosingBalanceData($transactions, $runningBalance, $sequenceNo); $processedData = $this->prepareProcessedClosingBalanceData($transactions, $runningBalance, $sequenceNo);
if (!empty($processedData)) { if (!empty($processedData)) {
foreach ($processedData as $data) { // Gunakan batch processing untuk updateOrCreate
ProcessedClosingBalance::updateOrCreate( $this->batchUpdateOrCreate($processedData);
[ }
'account_number' => $data['account_number'], });
'period' => $data['period'],
'trans_reference' => $data['trans_reference'],
'amount_lcy' => $data['amount_lcy'],
],
$data
);
}
}
});
DB::commit(); // HAPUS DB::commit() - akan di-handle di handle()
$recordCount = $this->getProcessedRecordCount(); $recordCount = $this->getProcessedRecordCount();
Log::info('Closing balance data processing completed successfully', [ Log::info('Closing balance data processing completed successfully', [
'final_sequence' => $sequenceNo, 'final_sequence' => $sequenceNo,
'final_balance' => $runningBalance, 'final_balance' => $runningBalance,
'record_count' => $recordCount 'record_count' => $recordCount
]); ]);
} catch (Exception $e) {
DB::rollback();
Log::error('Error in processAndSaveClosingBalanceData', [
'error' => $e->getMessage(),
'criteria' => $criteria
]);
throw $e;
}
} }
/** /**
@@ -601,4 +575,67 @@
'criteria' => $criteria 'criteria' => $criteria
]); ]);
} }
/**
* Batch update or create untuk mengurangi jumlah query dan lock
* Menggunakan pendekatan yang lebih efisien untuk menghindari max_lock_per_transaction
*/
private function batchUpdateOrCreate(array $processedData): void
{
Log::info('Starting batch updateOrCreate', [
'batch_size' => count($processedData)
]);
// Kumpulkan semua trans_reference yang akan diproses
$transReferences = collect($processedData)->pluck('trans_reference')->toArray();
// Ambil data yang sudah ada dalam satu query
$existingRecords = ProcessedClosingBalance::where('account_number', $this->accountNumber)
->where('period', $this->period)
->where('group_name', $this->groupName)
->whereIn('trans_reference', $transReferences)
->get()
->keyBy(function ($item) {
return $item->trans_reference . '_' . $item->amount_lcy;
});
$toInsert = [];
$toUpdate = [];
foreach ($processedData as $data) {
$key = $data['trans_reference'] . '_' . $data['amount_lcy'];
if ($existingRecords->has($key)) {
// Record sudah ada, siapkan untuk update
$existingRecord = $existingRecords->get($key);
$toUpdate[] = [
'id' => $existingRecord->id,
'data' => $data
];
} else {
// Record baru, siapkan untuk insert
$toInsert[] = $data;
}
}
// Batch insert untuk record baru
if (!empty($toInsert)) {
DB::table('processed_closing_balances')->insert($toInsert);
Log::info('Batch insert completed', ['count' => count($toInsert)]);
}
// Batch update untuk record yang sudah ada
if (!empty($toUpdate)) {
foreach ($toUpdate as $updateItem) {
ProcessedClosingBalance::where('id', $updateItem['id'])
->update($updateItem['data']);
}
Log::info('Batch update completed', ['count' => count($toUpdate)]);
}
Log::info('Batch updateOrCreate completed successfully', [
'inserted' => count($toInsert),
'updated' => count($toUpdate)
]);
}
} }