feat(closing-balance): Implementasi Pengecekan Unique Hash dan Sequence Counter pada Ekspor CSV

Memperbaiki logika ekspor CSV laporan penutupan saldo untuk memastikan konsistensi data dan urutan sequence yang benar.

Perubahan yang dilakukan:
• Menambahkan pengecekan unique_hash untuk mencegah duplikasi data dalam file CSV
• Mengimplementasikan array $processedHashes untuk melacak unique_hash yang sudah diproses
• Mengganti sequence_no dari database dengan counter yang bertambah ($sequenceCounter++)
• Menghapus groupBy('unique_hash') dari query untuk kontrol duplikasi yang lebih baik
• Menambahkan logging untuk record yang di-skip karena duplikasi unique_hash
• Menambahkan logging untuk tracking jumlah record yang diproses per chunk
• Menggunakan reference (&$sequenceCounter, &$processedHashes) untuk konsistensi data antar chunk
• Memastikan sequence number berurutan tanpa gap dimulai dari 1
This commit is contained in:
Daeng Deni Mardaeni
2025-07-31 11:55:55 +07:00
parent 710cbb5232
commit 8736ccf5f8

View File

@@ -2,20 +2,18 @@
namespace Modules\Webstatement\Jobs;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Queue\{InteractsWithQueue, SerializesModels};
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Support\Facades\{DB, Log, Storage};
use Carbon\Carbon;
use Exception;
use Modules\Webstatement\Models\{
AccountBalance,
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\{InteractsWithQueue, SerializesModels};
use Illuminate\Support\Facades\{DB, Log, Storage};
use Modules\Webstatement\Models\{AccountBalance,
ClosingBalanceReportLog,
ProcessedClosingBalance,
StmtEntry,
StmtEntryDetail
};
StmtEntryDetail};
/**
* Job untuk generate laporan closing balance dengan optimasi performa
@@ -46,7 +44,8 @@ class GenerateClosingBalanceReportJob implements ShouldQueue
/**
* Execute the job dengan optimasi performa
*/
public function handle(): void
public function handle()
: void
{
$reportLog = ClosingBalanceReportLog::find($this->reportLogId);
@@ -122,7 +121,8 @@ class GenerateClosingBalanceReportJob implements ShouldQueue
* Process and save closing balance data to database dengan proteksi duplikasi
* Memproses dan menyimpan data closing balance dengan perlindungan terhadap duplikasi
*/
private function processAndSaveClosingBalanceData(): void
private function processAndSaveClosingBalanceData()
: void
{
$criteria = [
'account_number' => $this->accountNumber,
@@ -215,7 +215,8 @@ class GenerateClosingBalanceReportJob implements ShouldQueue
* Delete existing processed data dengan verifikasi lengkap
* Menghapus data processed yang sudah ada dengan verifikasi untuk memastikan tidak ada sisa
*/
private function deleteExistingProcessedDataWithVerification(array $criteria): void
private function deleteExistingProcessedDataWithVerification(array $criteria)
: void
{
Log::info('Deleting existing processed data with verification', $criteria);
@@ -249,191 +250,12 @@ class GenerateClosingBalanceReportJob implements ShouldQueue
]);
}
/**
* Verify no duplicates after insert - simplified version
* Memverifikasi tidak ada duplikasi setelah insert data dengan pengecekan yang disederhanakan
*/
private function verifyNoDuplicatesAfterInsert(array $criteria): void
{
Log::info('Verifying no duplicates after insert with simplified check', $criteria);
// PERBAIKAN: Check for duplicate trans_reference + amount_lcy combinations saja
// Karena trans_reference sudah unique secara global, tidak perlu filter by account/period
$duplicates = DB::table('processed_closing_balances')
->select('trans_reference', 'amount_lcy', DB::raw('COUNT(*) as count'))
->where('account_number', $criteria['account_number'])
->where('period', $criteria['period'])
->where('group_name', $criteria['group_name'])
->groupBy('trans_reference', 'amount_lcy')
->having('count', '>', 1)
->get();
if ($duplicates->count() > 0) {
Log::error('Duplicates found after insert', [
'duplicate_count' => $duplicates->count(),
'duplicates' => $duplicates->toArray()
]);
throw new Exception("Duplicates detected after insert: {$duplicates->count()} duplicate combinations found");
}
// Check for duplicate sequence numbers dalam scope yang sama
$sequenceDuplicates = DB::table('processed_closing_balances')
->select('sequence_no', DB::raw('COUNT(*) as count'))
->where('account_number', $criteria['account_number'])
->where('period', $criteria['period'])
->where('group_name', $criteria['group_name'])
->groupBy('sequence_no')
->having('count', '>', 1)
->get();
if ($sequenceDuplicates->count() > 0) {
Log::error('Sequence number duplicates found after insert', [
'sequence_duplicate_count' => $sequenceDuplicates->count(),
'sequence_duplicates' => $sequenceDuplicates->toArray()
]);
throw new Exception("Sequence number duplicates detected: {$sequenceDuplicates->count()} duplicate sequences found");
}
Log::info('No duplicates found after insert - verification passed', $criteria);
}
/**
* Export from database to CSV (very fast)
*/
private function exportFromDatabaseToCsv(): string
{
Log::info('Starting CSV export from database for closing balance report', [
'account_number' => $this->accountNumber,
'period' => $this->period,
'group_name' => $this->groupName
]);
// Create directory structure
$basePath = "closing_balance_reports";
$accountPath = "{$basePath}/{$this->accountNumber}";
Storage::disk($this->disk)->makeDirectory($basePath);
Storage::disk($this->disk)->makeDirectory($accountPath);
// Generate filename
$fileName = "closing_balance_{$this->accountNumber}_{$this->period}_{$this->groupName}.csv";
$filePath = "{$accountPath}/{$fileName}";
// Delete existing file if exists
if (Storage::disk($this->disk)->exists($filePath)) {
Storage::disk($this->disk)->delete($filePath);
}
// Create CSV header
$csvHeader = [
'NO',
'TRANS_REFERENCE',
'BOOKING_DATE',
'TRANSACTION_DATE',
'AMOUNT_LCY',
'DEBIT_ACCT_NO',
'DEBIT_VALUE_DATE',
'DEBIT_AMOUNT',
'CREDIT_ACCT_NO',
'BIF_RCV_ACCT',
'BIF_RCV_NAME',
'CREDIT_VALUE_DATE',
'CREDIT_AMOUNT',
'AT_UNIQUE_ID',
'BIF_REF_NO',
'ATM_ORDER_ID',
'RECIPT_NO',
'API_ISS_ACCT',
'API_BENFF_ACCT',
'AUTHORISER',
'REMARKS',
'PAYMENT_DETAILS',
'REF_NO',
'MERCHANT_ID',
'TERM_ID',
'CLOSING_BALANCE'
];
$csvContent = implode('|', $csvHeader) . "\n";
Storage::disk($this->disk)->put($filePath, $csvContent);
// Export data from database in chunks dengan ordering berdasarkan booking_date dan date_time
ProcessedClosingBalance::where('account_number', $this->accountNumber)
->where('period', $this->period)
->where('group_name', $this->groupName)
->orderBy('booking_date')
->orderBy('transaction_date') // Menggunakan transaction_date yang sudah dikonversi dari date_time
->chunk($this->chunkSize, function ($records) use ($filePath) {
$csvContent = '';
foreach ($records as $record) {
$csvRow = [
$record->sequence_no,
$record->trans_reference ?? '',
$record->booking_date ?? '',
$record->transaction_date ?? '',
$record->amount_lcy ?? '',
$record->debit_acct_no ?? '',
$record->debit_value_date ?? '',
$record->debit_amount ?? '',
$record->credit_acct_no ?? '',
$record->bif_rcv_acct ?? '',
$record->bif_rcv_name ?? '',
$record->credit_value_date ?? '',
$record->credit_amount ?? '',
$record->at_unique_id ?? '',
$record->bif_ref_no ?? '',
$record->atm_order_id ?? '',
$record->recipt_no ?? '',
$record->api_iss_acct ?? '',
$record->api_benff_acct ?? '',
$record->authoriser ?? '',
$record->remarks ?? '',
$record->payment_details ?? '',
$record->ref_no ?? '',
$record->merchant_id ?? '',
$record->term_id ?? '',
$record->closing_balance ?? ''
];
$csvContent .= implode('|', $csvRow) . "\n";
}
if (!empty($csvContent)) {
Storage::disk($this->disk)->append($filePath, $csvContent);
}
});
// Verify file creation
if (!Storage::disk($this->disk)->exists($filePath)) {
throw new Exception("Failed to create CSV file: {$filePath}");
}
Log::info('CSV export from database completed successfully', [
'file_path' => $filePath,
'file_size' => Storage::disk($this->disk)->size($filePath)
]);
return $filePath;
}
/**
* Get processed record count
*/
private function getProcessedRecordCount(): int
{
return ProcessedClosingBalance::where('account_number', $this->accountNumber)
->where('period', $this->period)
->where('group_name', $this->groupName)
->count();
}
/**
* Get opening balance from account balance table
* Mengambil saldo awal dari tabel account balance
*/
private function getOpeningBalance(): float
private function getOpeningBalance()
: float
{
Log::info('Getting opening balance', [
'account_number' => $this->accountNumber,
@@ -521,11 +343,32 @@ class GenerateClosingBalanceReportJob implements ShouldQueue
return $query;
}
/**
* Get model class based on group name
* Mendapatkan class model berdasarkan group name
*/
private function getModelByGroup()
{
Log::info('Determining model by group', [
'group_name' => $this->groupName
]);
$model = $this->groupName === 'QRIS' ? StmtEntryDetail::class : StmtEntry::class;
Log::info('Model determined', [
'group_name' => $this->groupName,
'model_class' => $model
]);
return $model;
}
/**
* Prepare processed closing balance data dengan validasi duplikasi yang ketat
* Mempersiapkan data closing balance dengan pencegahan duplikasi berbasis trans_reference unik
*/
private function prepareProcessedClosingBalanceData($transactions, &$runningBalance, &$sequenceNo): array
private function prepareProcessedClosingBalanceData($transactions, &$runningBalance, &$sequenceNo)
: array
{
$processedData = [];
@@ -605,55 +448,12 @@ class GenerateClosingBalanceReportJob implements ShouldQueue
return $processedData;
}
/**
* Delete existing processed data dengan logging
* Menghapus data processed yang sudah ada dengan logging untuk audit
*/
private function deleteExistingProcessedData(array $criteria): void
{
Log::info('Deleting existing processed data', $criteria);
$deletedCount = ProcessedClosingBalance::where('account_number', $criteria['account_number'])
->where('period', $criteria['period'])
->where('group_name', $criteria['group_name'])
->count();
ProcessedClosingBalance::where('account_number', $criteria['account_number'])
->where('period', $criteria['period'])
->where('group_name', $criteria['group_name'])
->delete();
Log::info('Existing processed data deleted', [
'deleted_count' => $deletedCount,
'criteria' => $criteria
]);
}
/**
* Get model class based on group name
* Mendapatkan class model berdasarkan group name
*/
private function getModelByGroup()
{
Log::info('Determining model by group', [
'group_name' => $this->groupName
]);
$model = $this->groupName === 'QRIS' ? StmtEntryDetail::class : StmtEntry::class;
Log::info('Model determined', [
'group_name' => $this->groupName,
'model_class' => $model
]);
return $model;
}
/**
* Process transaction data from ORM result
* Memproses data transaksi dari hasil ORM
*/
private function processTransactionData($transaction): array
private function processTransactionData($transaction)
: array
{
Log::info('Processing transaction data', [
'trans_reference' => $transaction->trans_reference,
@@ -708,11 +508,262 @@ class GenerateClosingBalanceReportJob implements ShouldQueue
return $processedData;
}
/**
* Format datetime string
* Memformat string datetime
*/
private function formatDateTime(?string $datetime)
: string
{
if (!$datetime) {
return '';
}
try {
return Carbon::createFromFormat('ymdHi', $datetime)->format('d/m/Y H:i');
} catch (Exception $e) {
Log::warning('Error formatting datetime', [
'datetime' => $datetime,
'error' => $e->getMessage()
]);
return $datetime;
}
}
/**
* Verify no duplicates after insert - simplified version
* Memverifikasi tidak ada duplikasi setelah insert data dengan pengecekan yang disederhanakan
*/
private function verifyNoDuplicatesAfterInsert(array $criteria)
: void
{
Log::info('Verifying no duplicates after insert with simplified check', $criteria);
// PERBAIKAN: Check for duplicate trans_reference + amount_lcy combinations saja
// Karena trans_reference sudah unique secara global, tidak perlu filter by account/period
$duplicates = DB::table('processed_closing_balances')
->select('trans_reference', 'amount_lcy', DB::raw('COUNT(*) as count'))
->where('account_number', $criteria['account_number'])
->where('period', $criteria['period'])
->where('group_name', $criteria['group_name'])
->groupBy('trans_reference', 'amount_lcy')
->having('count', '>', 1)
->get();
if ($duplicates->count() > 0) {
Log::error('Duplicates found after insert', [
'duplicate_count' => $duplicates->count(),
'duplicates' => $duplicates->toArray()
]);
throw new Exception("Duplicates detected after insert: {$duplicates->count()} duplicate combinations found");
}
// Check for duplicate sequence numbers dalam scope yang sama
$sequenceDuplicates = DB::table('processed_closing_balances')
->select('sequence_no', DB::raw('COUNT(*) as count'))
->where('account_number', $criteria['account_number'])
->where('period', $criteria['period'])
->where('group_name', $criteria['group_name'])
->groupBy('sequence_no')
->having('count', '>', 1)
->get();
if ($sequenceDuplicates->count() > 0) {
Log::error('Sequence number duplicates found after insert', [
'sequence_duplicate_count' => $sequenceDuplicates->count(),
'sequence_duplicates' => $sequenceDuplicates->toArray()
]);
throw new Exception("Sequence number duplicates detected: {$sequenceDuplicates->count()} duplicate sequences found");
}
Log::info('No duplicates found after insert - verification passed', $criteria);
}
/**
* Export from database to CSV (very fast)
*/
private function exportFromDatabaseToCsv()
: string
{
Log::info('Starting CSV export from database for closing balance report', [
'account_number' => $this->accountNumber,
'period' => $this->period,
'group_name' => $this->groupName
]);
// Create directory structure
$basePath = "closing_balance_reports";
$accountPath = "{$basePath}/{$this->accountNumber}";
Storage::disk($this->disk)->makeDirectory($basePath);
Storage::disk($this->disk)->makeDirectory($accountPath);
// Generate filename
$fileName = "closing_balance_{$this->accountNumber}_{$this->period}_{$this->groupName}.csv";
$filePath = "{$accountPath}/{$fileName}";
// Delete existing file if exists
if (Storage::disk($this->disk)->exists($filePath)) {
Storage::disk($this->disk)->delete($filePath);
}
// Create CSV header
$csvHeader = [
'NO',
'TRANS_REFERENCE',
'BOOKING_DATE',
'TRANSACTION_DATE',
'AMOUNT_LCY',
'DEBIT_ACCT_NO',
'DEBIT_VALUE_DATE',
'DEBIT_AMOUNT',
'CREDIT_ACCT_NO',
'BIF_RCV_ACCT',
'BIF_RCV_NAME',
'CREDIT_VALUE_DATE',
'CREDIT_AMOUNT',
'AT_UNIQUE_ID',
'BIF_REF_NO',
'ATM_ORDER_ID',
'RECIPT_NO',
'API_ISS_ACCT',
'API_BENFF_ACCT',
'AUTHORISER',
'REMARKS',
'PAYMENT_DETAILS',
'REF_NO',
'MERCHANT_ID',
'TERM_ID',
'CLOSING_BALANCE'
];
$csvContent = implode('|', $csvHeader) . "\n";
Storage::disk($this->disk)->put($filePath, $csvContent);
// Inisialisasi counter untuk sequence number
$sequenceCounter = 1;
$processedHashes = [];
ProcessedClosingBalance::where('account_number', $this->accountNumber)
->where('period', $this->period)
->where('group_name', $this->groupName)
->orderBy('sequence_no')
->chunk($this->chunkSize, function ($records) use ($filePath, &$sequenceCounter, &$processedHashes, &$csvContent) {
foreach ($records as $record) {
// Pengecekan unique_hash: skip jika sudah diproses
if (in_array($record->unique_hash, $processedHashes)) {
Log::debug('Skipping duplicate unique_hash in CSV export', [
'unique_hash' => $record->unique_hash,
'trans_reference' => $record->trans_reference
]);
continue;
}
// Tandai unique_hash sebagai sudah diproses
$processedHashes[] = $record->unique_hash;
$csvRow = [
$sequenceCounter++,
// Gunakan counter yang bertambah, bukan sequence_no dari database
$record->trans_reference ?? '',
$record->booking_date ?? '',
$record->transaction_date ?? '',
$record->amount_lcy ?? '',
$record->debit_acct_no ?? '',
$record->debit_value_date ?? '',
$record->debit_amount ?? '',
$record->credit_acct_no ?? '',
$record->bif_rcv_acct ?? '',
$record->bif_rcv_name ?? '',
$record->credit_value_date ?? '',
$record->credit_amount ?? '',
$record->at_unique_id ?? '',
$record->bif_ref_no ?? '',
$record->atm_order_id ?? '',
$record->recipt_no ?? '',
$record->api_iss_acct ?? '',
$record->api_benff_acct ?? '',
$record->authoriser ?? '',
$record->remarks ?? '',
$record->payment_details ?? '',
$record->ref_no ?? '',
$record->merchant_id ?? '',
$record->term_id ?? '',
$record->closing_balance ?? ''
];
$csvContent .= implode('|', $csvRow) . "\n";
}
if (!empty($csvContent)) {
Storage::disk($this->disk)->append($filePath, $csvContent);
Log::debug('CSV content appended', [
'records_processed' => substr_count($csvContent, "\n"),
'current_sequence' => $sequenceCounter - 1
]);
}
});
// Verify file creation
if (!Storage::disk($this->disk)->exists($filePath)) {
throw new Exception("Failed to create CSV file: {$filePath}");
}
Log::info('CSV export from database completed successfully', [
'file_path' => $filePath,
'file_size' => Storage::disk($this->disk)->size($filePath)
]);
return $filePath;
}
/**
* Get processed record count
*/
private function getProcessedRecordCount()
: int
{
return ProcessedClosingBalance::where('account_number', $this->accountNumber)
->where('period', $this->period)
->where('group_name', $this->groupName)
->count();
}
/**
* Delete existing processed data dengan logging
* Menghapus data processed yang sudah ada dengan logging untuk audit
*/
private function deleteExistingProcessedData(array $criteria)
: void
{
Log::info('Deleting existing processed data', $criteria);
$deletedCount = ProcessedClosingBalance::where('account_number', $criteria['account_number'])
->where('period', $criteria['period'])
->where('group_name', $criteria['group_name'])
->count();
ProcessedClosingBalance::where('account_number', $criteria['account_number'])
->where('period', $criteria['period'])
->where('group_name', $criteria['group_name'])
->delete();
Log::info('Existing processed data deleted', [
'deleted_count' => $deletedCount,
'criteria' => $criteria
]);
}
/**
* Updated generateReportData method using pure ORM
* Method generateReportData yang diperbarui menggunakan ORM murni
*/
private function generateReportData(): array
private function generateReportData()
: array
{
Log::info('Starting report data generation using pure ORM', [
'account_number' => $this->accountNumber,
@@ -796,7 +847,8 @@ class GenerateClosingBalanceReportJob implements ShouldQueue
* Build report data row from transaction
* Membangun baris data laporan dari transaksi
*/
private function buildReportDataRow($transaction, int $sequenceNo, string $transactionDate, float $runningBalance): array
private function buildReportDataRow($transaction, int $sequenceNo, string $transactionDate, float $runningBalance)
: array
{
return [
'sequence_no' => $sequenceNo,
@@ -828,32 +880,12 @@ class GenerateClosingBalanceReportJob implements ShouldQueue
];
}
/**
* Format datetime string
* Memformat string datetime
*/
private function formatDateTime(?string $datetime): string
{
if (!$datetime) {
return '';
}
try {
return Carbon::createFromFormat('ymdHi', $datetime)->format('d/m/Y H:i');
} catch (Exception $e) {
Log::warning('Error formatting datetime', [
'datetime' => $datetime,
'error' => $e->getMessage()
]);
return $datetime;
}
}
/**
* Export report data to CSV file
* Export data laporan ke file CSV
*/
private function exportToCsv(array $reportData): string
private function exportToCsv(array $reportData)
: string
{
Log::info('Starting CSV export for closing balance report', [
'account_number' => $this->accountNumber,