daengdeni/webstatement
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(closing-balance): Implementasi Pengecekan Unique Hash dan Sequence Counter pada Ekspor CSV

Browse Source 
Memperbaiki logika ekspor CSV laporan penutupan saldo untuk memastikan konsistensi data dan urutan sequence yang benar.

Perubahan yang dilakukan:
• Menambahkan pengecekan unique_hash untuk mencegah duplikasi data dalam file CSV
• Mengimplementasikan array $processedHashes untuk melacak unique_hash yang sudah diproses
• Mengganti sequence_no dari database dengan counter yang bertambah ($sequenceCounter++)
• Menghapus groupBy('unique_hash') dari query untuk kontrol duplikasi yang lebih baik
• Menambahkan logging untuk record yang di-skip karena duplikasi unique_hash
• Menambahkan logging untuk tracking jumlah record yang diproses per chunk
• Menggunakan reference (&$sequenceCounter, &$processedHashes) untuk konsistensi data antar chunk
• Memastikan sequence number berurutan tanpa gap dimulai dari 1
This commit is contained in:
Daeng Deni Mardaeni
2025-07-31 11:55:55 +07:00
parent 710cbb5232
commit 8736ccf5f8
1 changed files with 923 additions and 891 deletions
Download Patch File Download Diff File Expand all files Collapse all files

578
app/Jobs/GenerateClosingBalanceReportJob.php
View File

@@ -1,28 +1,26 @@
<?php
namespace Modules\Webstatement\Jobs;
namespace Modules\Webstatement\Jobs;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Queue\{InteractsWithQueue, SerializesModels};
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Support\Facades\{DB, Log, Storage};
use Carbon\Carbon;
use Exception;
use Modules\Webstatement\Models\{
AccountBalance,
use Carbon\Carbon;
use Exception;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\{InteractsWithQueue, SerializesModels};
use Illuminate\Support\Facades\{DB, Log, Storage};
use Modules\Webstatement\Models\{AccountBalance,
ClosingBalanceReportLog,
ProcessedClosingBalance,
StmtEntry,
StmtEntryDetail
};
StmtEntryDetail};
/**
/**
* Job untuk generate laporan closing balance dengan optimasi performa
* Menggunakan database staging sebelum export CSV
*/
class GenerateClosingBalanceReportJob implements ShouldQueue
{
class GenerateClosingBalanceReportJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $accountNumber;
@@ -35,7 +33,7 @@ class GenerateClosingBalanceReportJob implements ShouldQueue
/**
* Create a new job instance.
*/
public function __construct(string $accountNumber, string $period, int $reportLogId, string $groupName='DEFAULT')
public function __construct(string $accountNumber, string $period, int $reportLogId, string $groupName = 'DEFAULT')
{
$this->accountNumber = $accountNumber;
$this->period = $period;
@@ -46,7 +44,8 @@ class GenerateClosingBalanceReportJob implements ShouldQueue
/**
* Execute the job dengan optimasi performa
*/
public function handle(): void
public function handle()
: void
{
$reportLog = ClosingBalanceReportLog::find($this->reportLogId);
@@ -122,7 +121,8 @@ class GenerateClosingBalanceReportJob implements ShouldQueue
* Process and save closing balance data to database dengan proteksi duplikasi
* Memproses dan menyimpan data closing balance dengan perlindungan terhadap duplikasi
*/
private function processAndSaveClosingBalanceData(): void
private function processAndSaveClosingBalanceData()
: void
{
$criteria = [
'account_number' => $this->accountNumber,
@@ -215,7 +215,8 @@ class GenerateClosingBalanceReportJob implements ShouldQueue
* Delete existing processed data dengan verifikasi lengkap
* Menghapus data processed yang sudah ada dengan verifikasi untuk memastikan tidak ada sisa
*/
private function deleteExistingProcessedDataWithVerification(array $criteria): void
private function deleteExistingProcessedDataWithVerification(array $criteria)
: void
{
Log::info('Deleting existing processed data with verification', $criteria);
@@ -249,191 +250,12 @@ class GenerateClosingBalanceReportJob implements ShouldQueue
]);
}
/**
* Verify no duplicates after insert - simplified version
* Memverifikasi tidak ada duplikasi setelah insert data dengan pengecekan yang disederhanakan
*/
private function verifyNoDuplicatesAfterInsert(array $criteria): void
{
Log::info('Verifying no duplicates after insert with simplified check', $criteria);
// PERBAIKAN: Check for duplicate trans_reference + amount_lcy combinations saja
// Karena trans_reference sudah unique secara global, tidak perlu filter by account/period
$duplicates = DB::table('processed_closing_balances')
->select('trans_reference', 'amount_lcy', DB::raw('COUNT(*) as count'))
->where('account_number', $criteria['account_number'])
->where('period', $criteria['period'])
->where('group_name', $criteria['group_name'])
->groupBy('trans_reference', 'amount_lcy')
->having('count', '>', 1)
->get();
if ($duplicates->count() > 0) {
Log::error('Duplicates found after insert', [
'duplicate_count' => $duplicates->count(),
'duplicates' => $duplicates->toArray()
]);
throw new Exception("Duplicates detected after insert: {$duplicates->count()} duplicate combinations found");
}
// Check for duplicate sequence numbers dalam scope yang sama
$sequenceDuplicates = DB::table('processed_closing_balances')
->select('sequence_no', DB::raw('COUNT(*) as count'))
->where('account_number', $criteria['account_number'])
->where('period', $criteria['period'])
->where('group_name', $criteria['group_name'])
->groupBy('sequence_no')
->having('count', '>', 1)
->get();
if ($sequenceDuplicates->count() > 0) {
Log::error('Sequence number duplicates found after insert', [
'sequence_duplicate_count' => $sequenceDuplicates->count(),
'sequence_duplicates' => $sequenceDuplicates->toArray()
]);
throw new Exception("Sequence number duplicates detected: {$sequenceDuplicates->count()} duplicate sequences found");
}
Log::info('No duplicates found after insert - verification passed', $criteria);
}
/**
* Export from database to CSV (very fast)
*/
private function exportFromDatabaseToCsv(): string
{
Log::info('Starting CSV export from database for closing balance report', [
'account_number' => $this->accountNumber,
'period' => $this->period,
'group_name' => $this->groupName
]);
// Create directory structure
$basePath = "closing_balance_reports";
$accountPath = "{$basePath}/{$this->accountNumber}";
Storage::disk($this->disk)->makeDirectory($basePath);
Storage::disk($this->disk)->makeDirectory($accountPath);
// Generate filename
$fileName = "closing_balance_{$this->accountNumber}_{$this->period}_{$this->groupName}.csv";
$filePath = "{$accountPath}/{$fileName}";
// Delete existing file if exists
if (Storage::disk($this->disk)->exists($filePath)) {
Storage::disk($this->disk)->delete($filePath);
}
// Create CSV header
$csvHeader = [
'NO',
'TRANS_REFERENCE',
'BOOKING_DATE',
'TRANSACTION_DATE',
'AMOUNT_LCY',
'DEBIT_ACCT_NO',
'DEBIT_VALUE_DATE',
'DEBIT_AMOUNT',
'CREDIT_ACCT_NO',
'BIF_RCV_ACCT',
'BIF_RCV_NAME',
'CREDIT_VALUE_DATE',
'CREDIT_AMOUNT',
'AT_UNIQUE_ID',
'BIF_REF_NO',
'ATM_ORDER_ID',
'RECIPT_NO',
'API_ISS_ACCT',
'API_BENFF_ACCT',
'AUTHORISER',
'REMARKS',
'PAYMENT_DETAILS',
'REF_NO',
'MERCHANT_ID',
'TERM_ID',
'CLOSING_BALANCE'
];
$csvContent = implode('|', $csvHeader) . "\n";
Storage::disk($this->disk)->put($filePath, $csvContent);
// Export data from database in chunks dengan ordering berdasarkan booking_date dan date_time
ProcessedClosingBalance::where('account_number', $this->accountNumber)
->where('period', $this->period)
->where('group_name', $this->groupName)
->orderBy('booking_date')
->orderBy('transaction_date') // Menggunakan transaction_date yang sudah dikonversi dari date_time
->chunk($this->chunkSize, function ($records) use ($filePath) {
$csvContent = '';
foreach ($records as $record) {
$csvRow = [
$record->sequence_no,
$record->trans_reference ?? '',
$record->booking_date ?? '',
$record->transaction_date ?? '',
$record->amount_lcy ?? '',
$record->debit_acct_no ?? '',
$record->debit_value_date ?? '',
$record->debit_amount ?? '',
$record->credit_acct_no ?? '',
$record->bif_rcv_acct ?? '',
$record->bif_rcv_name ?? '',
$record->credit_value_date ?? '',
$record->credit_amount ?? '',
$record->at_unique_id ?? '',
$record->bif_ref_no ?? '',
$record->atm_order_id ?? '',
$record->recipt_no ?? '',
$record->api_iss_acct ?? '',
$record->api_benff_acct ?? '',
$record->authoriser ?? '',
$record->remarks ?? '',
$record->payment_details ?? '',
$record->ref_no ?? '',
$record->merchant_id ?? '',
$record->term_id ?? '',
$record->closing_balance ?? ''
];
$csvContent .= implode('|', $csvRow) . "\n";
}
if (!empty($csvContent)) {
Storage::disk($this->disk)->append($filePath, $csvContent);
}
});
// Verify file creation
if (!Storage::disk($this->disk)->exists($filePath)) {
throw new Exception("Failed to create CSV file: {$filePath}");
}
Log::info('CSV export from database completed successfully', [
'file_path' => $filePath,
'file_size' => Storage::disk($this->disk)->size($filePath)
]);
return $filePath;
}
/**
* Get processed record count
*/
private function getProcessedRecordCount(): int
{
return ProcessedClosingBalance::where('account_number', $this->accountNumber)
->where('period', $this->period)
->where('group_name', $this->groupName)
->count();
}
/**
* Get opening balance from account balance table
* Mengambil saldo awal dari tabel account balance
*/
private function getOpeningBalance(): float
private function getOpeningBalance()
: float
{
Log::info('Getting opening balance', [
'account_number' => $this->accountNumber,
@@ -503,14 +325,14 @@ class GenerateClosingBalanceReportJob implements ShouldQueue
'date_time'
])
->with([
'ft' => function($query) {
'ft' => function ($query) {
$query->select('ref_no', 'date_time', 'debit_acct_no', 'debit_value_date',
'credit_acct_no', 'bif_rcv_acct', 'bif_rcv_name', 'credit_value_date',
'at_unique_id', 'bif_ref_no', 'atm_order_id', 'recipt_no',
'api_iss_acct', 'api_benff_acct', 'authoriser', 'remarks',
'payment_details', 'ref_no', 'merchant_id', 'term_id');
},
'dc' => function($query) {
'dc' => function ($query) {
$query->select('id', 'date_time');
}
])
@@ -521,11 +343,32 @@ class GenerateClosingBalanceReportJob implements ShouldQueue
return $query;
}
/**
* Get model class based on group name
* Mendapatkan class model berdasarkan group name
*/
private function getModelByGroup()
{
Log::info('Determining model by group', [
'group_name' => $this->groupName
]);
$model = $this->groupName === 'QRIS' ? StmtEntryDetail::class : StmtEntry::class;
Log::info('Model determined', [
'group_name' => $this->groupName,
'model_class' => $model
]);
return $model;
}
/**
* Prepare processed closing balance data dengan validasi duplikasi yang ketat
* Mempersiapkan data closing balance dengan pencegahan duplikasi berbasis trans_reference unik
*/
private function prepareProcessedClosingBalanceData($transactions, &$runningBalance, &$sequenceNo): array
private function prepareProcessedClosingBalanceData($transactions, &$runningBalance, &$sequenceNo)
: array
{
$processedData = [];
@@ -605,55 +448,12 @@ class GenerateClosingBalanceReportJob implements ShouldQueue
return $processedData;
}
/**
* Delete existing processed data dengan logging
* Menghapus data processed yang sudah ada dengan logging untuk audit
*/
private function deleteExistingProcessedData(array $criteria): void
{
Log::info('Deleting existing processed data', $criteria);
$deletedCount = ProcessedClosingBalance::where('account_number', $criteria['account_number'])
->where('period', $criteria['period'])
->where('group_name', $criteria['group_name'])
->count();
ProcessedClosingBalance::where('account_number', $criteria['account_number'])
->where('period', $criteria['period'])
->where('group_name', $criteria['group_name'])
->delete();
Log::info('Existing processed data deleted', [
'deleted_count' => $deletedCount,
'criteria' => $criteria
]);
}
/**
* Get model class based on group name
* Mendapatkan class model berdasarkan group name
*/
private function getModelByGroup()
{
Log::info('Determining model by group', [
'group_name' => $this->groupName
]);
$model = $this->groupName === 'QRIS' ? StmtEntryDetail::class : StmtEntry::class;
Log::info('Model determined', [
'group_name' => $this->groupName,
'model_class' => $model
]);
return $model;
}
/**
* Process transaction data from ORM result
* Memproses data transaksi dari hasil ORM
*/
private function processTransactionData($transaction): array
private function processTransactionData($transaction)
: array
{
Log::info('Processing transaction data', [
'trans_reference' => $transaction->trans_reference,
@@ -708,11 +508,262 @@ class GenerateClosingBalanceReportJob implements ShouldQueue
return $processedData;
}
/**
* Format datetime string
* Memformat string datetime
*/
private function formatDateTime(?string $datetime)
: string
{
if (!$datetime) {
return '';
}
try {
return Carbon::createFromFormat('ymdHi', $datetime)->format('d/m/Y H:i');
} catch (Exception $e) {
Log::warning('Error formatting datetime', [
'datetime' => $datetime,
'error' => $e->getMessage()
]);
return $datetime;
}
}
/**
* Verify no duplicates after insert - simplified version
* Memverifikasi tidak ada duplikasi setelah insert data dengan pengecekan yang disederhanakan
*/
private function verifyNoDuplicatesAfterInsert(array $criteria)
: void
{
Log::info('Verifying no duplicates after insert with simplified check', $criteria);
// PERBAIKAN: Check for duplicate trans_reference + amount_lcy combinations saja
// Karena trans_reference sudah unique secara global, tidak perlu filter by account/period
$duplicates = DB::table('processed_closing_balances')
->select('trans_reference', 'amount_lcy', DB::raw('COUNT(*) as count'))
->where('account_number', $criteria['account_number'])
->where('period', $criteria['period'])
->where('group_name', $criteria['group_name'])
->groupBy('trans_reference', 'amount_lcy')
->having('count', '>', 1)
->get();
if ($duplicates->count() > 0) {
Log::error('Duplicates found after insert', [
'duplicate_count' => $duplicates->count(),
'duplicates' => $duplicates->toArray()
]);
throw new Exception("Duplicates detected after insert: {$duplicates->count()} duplicate combinations found");
}
// Check for duplicate sequence numbers dalam scope yang sama
$sequenceDuplicates = DB::table('processed_closing_balances')
->select('sequence_no', DB::raw('COUNT(*) as count'))
->where('account_number', $criteria['account_number'])
->where('period', $criteria['period'])
->where('group_name', $criteria['group_name'])
->groupBy('sequence_no')
->having('count', '>', 1)
->get();
if ($sequenceDuplicates->count() > 0) {
Log::error('Sequence number duplicates found after insert', [
'sequence_duplicate_count' => $sequenceDuplicates->count(),
'sequence_duplicates' => $sequenceDuplicates->toArray()
]);
throw new Exception("Sequence number duplicates detected: {$sequenceDuplicates->count()} duplicate sequences found");
}
Log::info('No duplicates found after insert - verification passed', $criteria);
}
/**
* Export from database to CSV (very fast)
*/
private function exportFromDatabaseToCsv()
: string
{
Log::info('Starting CSV export from database for closing balance report', [
'account_number' => $this->accountNumber,
'period' => $this->period,
'group_name' => $this->groupName
]);
// Create directory structure
$basePath = "closing_balance_reports";
$accountPath = "{$basePath}/{$this->accountNumber}";
Storage::disk($this->disk)->makeDirectory($basePath);
Storage::disk($this->disk)->makeDirectory($accountPath);
// Generate filename
$fileName = "closing_balance_{$this->accountNumber}_{$this->period}_{$this->groupName}.csv";
$filePath = "{$accountPath}/{$fileName}";
// Delete existing file if exists
if (Storage::disk($this->disk)->exists($filePath)) {
Storage::disk($this->disk)->delete($filePath);
}
// Create CSV header
$csvHeader = [
'NO',
'TRANS_REFERENCE',
'BOOKING_DATE',
'TRANSACTION_DATE',
'AMOUNT_LCY',
'DEBIT_ACCT_NO',
'DEBIT_VALUE_DATE',
'DEBIT_AMOUNT',
'CREDIT_ACCT_NO',
'BIF_RCV_ACCT',
'BIF_RCV_NAME',
'CREDIT_VALUE_DATE',
'CREDIT_AMOUNT',
'AT_UNIQUE_ID',
'BIF_REF_NO',
'ATM_ORDER_ID',
'RECIPT_NO',
'API_ISS_ACCT',
'API_BENFF_ACCT',
'AUTHORISER',
'REMARKS',
'PAYMENT_DETAILS',
'REF_NO',
'MERCHANT_ID',
'TERM_ID',
'CLOSING_BALANCE'
];
$csvContent = implode('|', $csvHeader) . "\n";
Storage::disk($this->disk)->put($filePath, $csvContent);
// Inisialisasi counter untuk sequence number
$sequenceCounter = 1;
$processedHashes = [];
ProcessedClosingBalance::where('account_number', $this->accountNumber)
->where('period', $this->period)
->where('group_name', $this->groupName)
->orderBy('sequence_no')
->chunk($this->chunkSize, function ($records) use ($filePath, &$sequenceCounter, &$processedHashes, &$csvContent) {
foreach ($records as $record) {
// Pengecekan unique_hash: skip jika sudah diproses
if (in_array($record->unique_hash, $processedHashes)) {
Log::debug('Skipping duplicate unique_hash in CSV export', [
'unique_hash' => $record->unique_hash,
'trans_reference' => $record->trans_reference
]);
continue;
}
// Tandai unique_hash sebagai sudah diproses
$processedHashes[] = $record->unique_hash;
$csvRow = [
$sequenceCounter++,
// Gunakan counter yang bertambah, bukan sequence_no dari database
$record->trans_reference ?? '',
$record->booking_date ?? '',
$record->transaction_date ?? '',
$record->amount_lcy ?? '',
$record->debit_acct_no ?? '',
$record->debit_value_date ?? '',
$record->debit_amount ?? '',
$record->credit_acct_no ?? '',
$record->bif_rcv_acct ?? '',
$record->bif_rcv_name ?? '',
$record->credit_value_date ?? '',
$record->credit_amount ?? '',
$record->at_unique_id ?? '',
$record->bif_ref_no ?? '',
$record->atm_order_id ?? '',
$record->recipt_no ?? '',
$record->api_iss_acct ?? '',
$record->api_benff_acct ?? '',
$record->authoriser ?? '',
$record->remarks ?? '',
$record->payment_details ?? '',
$record->ref_no ?? '',
$record->merchant_id ?? '',
$record->term_id ?? '',
$record->closing_balance ?? ''
];
$csvContent .= implode('|', $csvRow) . "\n";
}
if (!empty($csvContent)) {
Storage::disk($this->disk)->append($filePath, $csvContent);
Log::debug('CSV content appended', [
'records_processed' => substr_count($csvContent, "\n"),
'current_sequence' => $sequenceCounter - 1
]);
}
});
// Verify file creation
if (!Storage::disk($this->disk)->exists($filePath)) {
throw new Exception("Failed to create CSV file: {$filePath}");
}
Log::info('CSV export from database completed successfully', [
'file_path' => $filePath,
'file_size' => Storage::disk($this->disk)->size($filePath)
]);
return $filePath;
}
/**
* Get processed record count
*/
private function getProcessedRecordCount()
: int
{
return ProcessedClosingBalance::where('account_number', $this->accountNumber)
->where('period', $this->period)
->where('group_name', $this->groupName)
->count();
}
/**
* Delete existing processed data dengan logging
* Menghapus data processed yang sudah ada dengan logging untuk audit
*/
private function deleteExistingProcessedData(array $criteria)
: void
{
Log::info('Deleting existing processed data', $criteria);
$deletedCount = ProcessedClosingBalance::where('account_number', $criteria['account_number'])
->where('period', $criteria['period'])
->where('group_name', $criteria['group_name'])
->count();
ProcessedClosingBalance::where('account_number', $criteria['account_number'])
->where('period', $criteria['period'])
->where('group_name', $criteria['group_name'])
->delete();
Log::info('Existing processed data deleted', [
'deleted_count' => $deletedCount,
'criteria' => $criteria
]);
}
/**
* Updated generateReportData method using pure ORM
* Method generateReportData yang diperbarui menggunakan ORM murni
*/
private function generateReportData(): array
private function generateReportData()
: array
{
Log::info('Starting report data generation using pure ORM', [
'account_number' => $this->accountNumber,
@@ -796,7 +847,8 @@ class GenerateClosingBalanceReportJob implements ShouldQueue
* Build report data row from transaction
* Membangun baris data laporan dari transaksi
*/
private function buildReportDataRow($transaction, int $sequenceNo, string $transactionDate, float $runningBalance): array
private function buildReportDataRow($transaction, int $sequenceNo, string $transactionDate, float $runningBalance)
: array
{
return [
'sequence_no' => $sequenceNo,
@@ -828,32 +880,12 @@ class GenerateClosingBalanceReportJob implements ShouldQueue
];
}
/**
* Format datetime string
* Memformat string datetime
*/
private function formatDateTime(?string $datetime): string
{
if (!$datetime) {
return '';
}
try {
return Carbon::createFromFormat('ymdHi', $datetime)->format('d/m/Y H:i');
} catch (Exception $e) {
Log::warning('Error formatting datetime', [
'datetime' => $datetime,
'error' => $e->getMessage()
]);
return $datetime;
}
}
/**
* Export report data to CSV file
* Export data laporan ke file CSV
*/
private function exportToCsv(array $reportData): string
private function exportToCsv(array $reportData)
: string
{
Log::info('Starting CSV export for closing balance report', [
'account_number' => $this->accountNumber,
@@ -958,4 +990,4 @@ class GenerateClosingBalanceReportJob implements ShouldQueue
return $filePath;
}
}
}