accountNumber = $accountNumber; $this->period = $period; $this->reportLogId = $reportLogId; $this->groupName = $groupName ?? 'DEFAULT'; } /** * Execute the job dengan optimasi performa */ public function handle(): void { $reportLog = ClosingBalanceReportLog::find($this->reportLogId); if (!$reportLog) { Log::error('Closing balance report log not found', ['id' => $this->reportLogId]); return; } try { Log::info('Starting optimized closing balance report generation', [ 'account_number' => $this->accountNumber, 'period' => $this->period, 'group_name' => $this->groupName, 'report_log_id' => $this->reportLogId ]); DB::beginTransaction(); // Update status to processing $reportLog->update([ 'status' => 'processing', 'updated_at' => now() ]); // Step 1: Process and save to database (fast) $this->processAndSaveClosingBalanceData(); // Step 2: Export from database to CSV (fast) $filePath = $this->exportFromDatabaseToCsv(); // Get record count from database $recordCount = $this->getProcessedRecordCount(); // Update report log with success $reportLog->update([ 'status' => 'completed', 'file_path' => $filePath, 'file_size' => Storage::disk($this->disk)->size($filePath), 'record_count' => $recordCount, 'updated_at' => now() ]); DB::commit(); Log::info('Optimized closing balance report generation completed successfully', [ 'account_number' => $this->accountNumber, 'period' => $this->period, 'file_path' => $filePath, 'record_count' => $recordCount ]); } catch (Exception $e) { DB::rollback(); Log::error('Error generating optimized closing balance report', [ 'account_number' => $this->accountNumber, 'period' => $this->period, 'error' => $e->getMessage(), 'trace' => $e->getTraceAsString() ]); $reportLog->update([ 'status' => 'failed', 'error_message' => $e->getMessage(), 'updated_at' => now() ]); throw $e; } } /** * Process and save closing balance data to database dengan proteksi duplikasi * Memproses dan menyimpan data closing balance dengan perlindungan terhadap duplikasi */ private function processAndSaveClosingBalanceData(): void { $criteria = [ 'account_number' => $this->accountNumber, 'period' => $this->period, 'group_name' => $this->groupName ]; // PERBAIKAN: Gunakan database transaction untuk memastikan atomicity DB::beginTransaction(); try { // PERBAIKAN: Lock table untuk mencegah race condition DB::statement('LOCK TABLE processed_closing_balances IN EXCLUSIVE MODE'); // Delete existing processed data dengan verifikasi $this->deleteExistingProcessedDataWithVerification($criteria); // Get opening balance $runningBalance = $this->getOpeningBalance(); $sequenceNo = 0; $totalProcessed = 0; Log::info('Starting to process and save closing balance data with duplicate protection', [ 'opening_balance' => $runningBalance, 'criteria' => $criteria ]); // Build query $query = $this->buildTransactionQuery(); // PERBAIKAN: Collect all data first, then insert in single transaction $allProcessedData = []; $query->chunk($this->chunkSize, function ($transactions) use (&$runningBalance, &$sequenceNo, &$allProcessedData, &$totalProcessed) { $processedData = $this->prepareProcessedClosingBalanceData($transactions, $runningBalance, $sequenceNo); if (!empty($processedData)) { $allProcessedData = array_merge($allProcessedData, $processedData); $totalProcessed += count($processedData); } Log::info('Chunk processed and collected', [ 'chunk_size' => count($processedData), 'total_collected' => count($allProcessedData), 'current_balance' => $runningBalance ]); }); // PERBAIKAN: Insert all data in single operation if (!empty($allProcessedData)) { // Batch insert dengan chunk untuk menghindari memory limit $insertChunks = array_chunk($allProcessedData, 1000); foreach ($insertChunks as $chunk) { DB::table('processed_closing_balances')->insert($chunk); } Log::info('All processed data inserted successfully', [ 'total_records' => count($allProcessedData), 'insert_chunks' => count($insertChunks) ]); } // PERBAIKAN: Verify no duplicates after insert $this->verifyNoDuplicatesAfterInsert($criteria); DB::commit(); Log::info('Closing balance data processing completed successfully', [ 'final_sequence' => $sequenceNo, 'final_balance' => $runningBalance, 'total_processed' => $totalProcessed ]); } catch (Exception $e) { DB::rollback(); Log::error('Error in processAndSaveClosingBalanceData', [ 'error' => $e->getMessage(), 'criteria' => $criteria ]); throw $e; } } /** * Delete existing processed data dengan verifikasi lengkap * Menghapus data processed yang sudah ada dengan verifikasi untuk memastikan tidak ada sisa */ private function deleteExistingProcessedDataWithVerification(array $criteria): void { Log::info('Deleting existing processed data with verification', $criteria); // Count before deletion $beforeCount = ProcessedClosingBalance::where('account_number', $criteria['account_number']) ->where('period', $criteria['period']) ->where('group_name', $criteria['group_name']) ->count(); // Delete with force $deletedCount = ProcessedClosingBalance::where('account_number', $criteria['account_number']) ->where('period', $criteria['period']) ->where('group_name', $criteria['group_name']) ->delete(); // Verify deletion $afterCount = ProcessedClosingBalance::where('account_number', $criteria['account_number']) ->where('period', $criteria['period']) ->where('group_name', $criteria['group_name']) ->count(); if ($afterCount > 0) { throw new Exception("Failed to delete all existing data. Remaining records: {$afterCount}"); } Log::info('Existing processed data deleted and verified', [ 'before_count' => $beforeCount, 'deleted_count' => $deletedCount, 'after_count' => $afterCount, 'criteria' => $criteria ]); } /** * Verify no duplicates after insert - simplified version * Memverifikasi tidak ada duplikasi setelah insert data dengan pengecekan yang disederhanakan */ private function verifyNoDuplicatesAfterInsert(array $criteria): void { Log::info('Verifying no duplicates after insert with simplified check', $criteria); // PERBAIKAN: Check for duplicate trans_reference + amount_lcy combinations saja // Karena trans_reference sudah unique secara global, tidak perlu filter by account/period $duplicates = DB::table('processed_closing_balances') ->select('trans_reference', 'amount_lcy', DB::raw('COUNT(*) as count')) ->where('account_number', $criteria['account_number']) ->where('period', $criteria['period']) ->where('group_name', $criteria['group_name']) ->groupBy('trans_reference', 'amount_lcy') ->having('count', '>', 1) ->get(); if ($duplicates->count() > 0) { Log::error('Duplicates found after insert', [ 'duplicate_count' => $duplicates->count(), 'duplicates' => $duplicates->toArray() ]); throw new Exception("Duplicates detected after insert: {$duplicates->count()} duplicate combinations found"); } // Check for duplicate sequence numbers dalam scope yang sama $sequenceDuplicates = DB::table('processed_closing_balances') ->select('sequence_no', DB::raw('COUNT(*) as count')) ->where('account_number', $criteria['account_number']) ->where('period', $criteria['period']) ->where('group_name', $criteria['group_name']) ->groupBy('sequence_no') ->having('count', '>', 1) ->get(); if ($sequenceDuplicates->count() > 0) { Log::error('Sequence number duplicates found after insert', [ 'sequence_duplicate_count' => $sequenceDuplicates->count(), 'sequence_duplicates' => $sequenceDuplicates->toArray() ]); throw new Exception("Sequence number duplicates detected: {$sequenceDuplicates->count()} duplicate sequences found"); } Log::info('No duplicates found after insert - verification passed', $criteria); } /** * Build transaction query dengan eliminasi duplicate yang efektif * Membangun query transaksi dengan menghilangkan duplicate berdasarkan trans_reference dan amount_lcy */ private function buildTransactionQuery() { Log::info('Building transaction query with effective duplicate elimination', [ 'group_name' => $this->groupName, 'account_number' => $this->accountNumber, 'period' => $this->period ]); $modelClass = $this->getModelByGroup(); $tableName = (new $modelClass)->getTable(); // PERBAIKAN: Gunakan subquery untuk mendapatkan ID unik berdasarkan trans_reference + amount_lcy // Karena trans_reference sudah unique, fokus pada kombinasi trans_reference + amount_lcy $uniqueIds = DB::table($tableName) ->select(DB::raw('MIN(id) as min_id')) ->where('account_number', $this->accountNumber) ->where('booking_date', $this->period) ->groupBy('trans_reference', 'amount_lcy') // Simplified grouping ->pluck('min_id'); Log::info('Unique transaction IDs identified based on trans_reference + amount_lcy', [ 'total_unique_transactions' => $uniqueIds->count() ]); // Query hanya transaksi dengan ID yang unik $query = $modelClass::select([ 'id', 'trans_reference', 'booking_date', 'amount_lcy', 'date_time' ]) ->with([ 'ft' => function($query) { $query->select('ref_no', 'date_time', 'debit_acct_no', 'debit_value_date', 'credit_acct_no', 'bif_rcv_acct', 'bif_rcv_name', 'credit_value_date', 'at_unique_id', 'bif_ref_no', 'atm_order_id', 'recipt_no', 'api_iss_acct', 'api_benff_acct', 'authoriser', 'remarks', 'payment_details', 'ref_no', 'merchant_id', 'term_id'); }, 'dc' => function($query) { $query->select('id', 'date_time'); } ]) ->whereIn('id', $uniqueIds) ->orderBy('booking_date') ->orderBy('date_time'); Log::info('Transaction query built successfully with simplified duplicate elimination', [ 'model_class' => $modelClass, 'table_name' => $tableName, 'unique_transactions' => $uniqueIds->count() ]); return $query; } /** * Prepare processed closing balance data tanpa validasi duplicate (sudah dieliminasi di query) * Mempersiapkan data closing balance tanpa perlu validasi duplicate lagi */ private function prepareProcessedClosingBalanceData($transactions, &$runningBalance, &$sequenceNo): array { $processedData = []; foreach ($transactions as $transaction) { $sequenceNo++; // Process transaction data $processedTransactionData = $this->processTransactionData($transaction); // Update running balance $amount = (float) $transaction->amount_lcy; $runningBalance += $amount; // Format transaction date $transactionDate = $this->formatDateTime($processedTransactionData['date_time']); // Prepare data for database insert $processedData[] = [ 'account_number' => $this->accountNumber, 'period' => $this->period, 'group_name' => $this->groupName, 'sequence_no' => $sequenceNo, 'trans_reference' => $processedTransactionData['trans_reference'], 'booking_date' => $processedTransactionData['booking_date'], 'transaction_date' => $transactionDate, 'amount_lcy' => $processedTransactionData['amount_lcy'], 'debit_acct_no' => $processedTransactionData['debit_acct_no'], 'debit_value_date' => $processedTransactionData['debit_value_date'], 'debit_amount' => $processedTransactionData['debit_amount'], 'credit_acct_no' => $processedTransactionData['credit_acct_no'], 'bif_rcv_acct' => $processedTransactionData['bif_rcv_acct'], 'bif_rcv_name' => $processedTransactionData['bif_rcv_name'], 'credit_value_date' => $processedTransactionData['credit_value_date'], 'credit_amount' => $processedTransactionData['credit_amount'], 'at_unique_id' => $processedTransactionData['at_unique_id'], 'bif_ref_no' => $processedTransactionData['bif_ref_no'], 'atm_order_id' => $processedTransactionData['atm_order_id'], 'recipt_no' => $processedTransactionData['recipt_no'], 'api_iss_acct' => $processedTransactionData['api_iss_acct'], 'api_benff_acct' => $processedTransactionData['api_benff_acct'], 'authoriser' => $processedTransactionData['authoriser'], 'remarks' => $processedTransactionData['remarks'], 'payment_details' => $processedTransactionData['payment_details'], 'ref_no' => $processedTransactionData['ref_no'], 'merchant_id' => $processedTransactionData['merchant_id'], 'term_id' => $processedTransactionData['term_id'], 'closing_balance' => $runningBalance, 'created_at' => now(), 'updated_at' => now() ]; } Log::info('Processed closing balance data prepared without duplicates', [ 'total_records' => count($processedData) ]); return $processedData; } /** * Delete existing processed data dengan logging * Menghapus data processed yang sudah ada dengan logging untuk audit */ private function deleteExistingProcessedData(array $criteria): void { Log::info('Deleting existing processed data', $criteria); $deletedCount = ProcessedClosingBalance::where('account_number', $criteria['account_number']) ->where('period', $criteria['period']) ->where('group_name', $criteria['group_name']) ->count(); ProcessedClosingBalance::where('account_number', $criteria['account_number']) ->where('period', $criteria['period']) ->where('group_name', $criteria['group_name']) ->delete(); Log::info('Existing processed data deleted', [ 'deleted_count' => $deletedCount, 'criteria' => $criteria ]); } /** * Get model class based on group name * Mendapatkan class model berdasarkan group name */ private function getModelByGroup() { Log::info('Determining model by group', [ 'group_name' => $this->groupName ]); $model = $this->groupName === 'QRIS' ? StmtEntryDetail::class : StmtEntry::class; Log::info('Model determined', [ 'group_name' => $this->groupName, 'model_class' => $model ]); return $model; } /** * Process transaction data from ORM result * Memproses data transaksi dari hasil ORM */ private function processTransactionData($transaction): array { Log::info('Processing transaction data', [ 'trans_reference' => $transaction->trans_reference, 'has_ft_relation' => !is_null($transaction->ft), 'has_dc_relation' => !is_null($transaction->dc) ]); // Hitung debit dan credit amount $debitAmount = $transaction->amount_lcy < 0 ? abs($transaction->amount_lcy) : null; $creditAmount = $transaction->amount_lcy > 0 ? $transaction->amount_lcy : null; // Ambil date_time dari prioritas: ft -> dc -> stmt $dateTime = $transaction->ft?->date_time ?? $transaction->dc?->date_time ?? $transaction->date_time; $processedData = [ 'trans_reference' => $transaction->trans_reference, 'booking_date' => $transaction->booking_date, 'amount_lcy' => $transaction->amount_lcy, 'debit_amount' => $debitAmount, 'credit_amount' => $creditAmount, 'date_time' => $dateTime, // Data dari TempFundsTransfer melalui relasi 'debit_acct_no' => $transaction->ft?->debit_acct_no, 'debit_value_date' => $transaction->ft?->debit_value_date, 'credit_acct_no' => $transaction->ft?->credit_acct_no, 'bif_rcv_acct' => $transaction->ft?->bif_rcv_acct, 'bif_rcv_name' => $transaction->ft?->bif_rcv_name, 'credit_value_date' => $transaction->ft?->credit_value_date, 'at_unique_id' => $transaction->ft?->at_unique_id, 'bif_ref_no' => $transaction->ft?->bif_ref_no, 'atm_order_id' => $transaction->ft?->atm_order_id, 'recipt_no' => $transaction->ft?->recipt_no, 'api_iss_acct' => $transaction->ft?->api_iss_acct, 'api_benff_acct' => $transaction->ft?->api_benff_acct, 'authoriser' => $transaction->ft?->authoriser, 'remarks' => $transaction->ft?->remarks, 'payment_details' => $transaction->ft?->payment_details, 'ref_no' => $transaction->ft?->ref_no, 'merchant_id' => $transaction->ft?->merchant_id, 'term_id' => $transaction->ft?->term_id, ]; Log::info('Transaction data processed successfully', [ 'trans_reference' => $transaction->trans_reference, 'final_date_time' => $dateTime, 'debit_amount' => $debitAmount, 'credit_amount' => $creditAmount ]); return $processedData; } /** * Updated generateReportData method using pure ORM * Method generateReportData yang diperbarui menggunakan ORM murni */ private function generateReportData(): array { Log::info('Starting report data generation using pure ORM', [ 'account_number' => $this->accountNumber, 'period' => $this->period, 'group_name' => $this->groupName, 'chunk_size' => $this->chunkSize ]); $reportData = []; $runningBalance = $this->getOpeningBalance(); $sequenceNo = 1; try { DB::beginTransaction(); // Build query menggunakan pure ORM $query = $this->buildTransactionQuery(); // Process data dalam chunks untuk efisiensi memory $query->chunk($this->chunkSize, function ($transactions) use (&$reportData, &$runningBalance, &$sequenceNo) { Log::info('Processing transaction chunk', [ 'chunk_size' => $transactions->count(), 'current_sequence' => $sequenceNo, 'current_balance' => $runningBalance ]); foreach ($transactions as $transaction) { // Process transaction data $processedData = $this->processTransactionData($transaction); // Update running balance $amount = (float) $transaction->amount_lcy; $runningBalance += $amount; // Format transaction date $transactionDate = $this->formatDateTime($processedData['date_time']); // Build report data row $reportData[] = $this->buildReportDataRow( (object) $processedData, $sequenceNo, $transactionDate, $runningBalance ); $sequenceNo++; } Log::info('Chunk processed successfully', [ 'processed_count' => $transactions->count(), 'total_records_so_far' => count($reportData), 'current_balance' => $runningBalance ]); }); DB::commit(); Log::info('Report data generation completed using pure ORM', [ 'total_records' => count($reportData), 'final_balance' => $runningBalance, 'final_sequence' => $sequenceNo - 1 ]); } catch (Exception $e) { DB::rollback(); Log::error('Error generating report data using pure ORM', [ 'error' => $e->getMessage(), 'trace' => $e->getTraceAsString(), 'account_number' => $this->accountNumber, 'period' => $this->period ]); throw $e; } return $reportData; } /** * Build report data row from transaction * Membangun baris data laporan dari transaksi */ private function buildReportDataRow($transaction, int $sequenceNo, string $transactionDate, float $runningBalance): array { return [ 'sequence_no' => $sequenceNo, 'trans_reference' => $transaction->trans_reference, 'booking_date' => $transaction->booking_date, 'transaction_date' => $transactionDate, 'amount_lcy' => $transaction->amount_lcy, 'debit_acct_no' => $transaction->debit_acct_no, 'debit_value_date' => $transaction->debit_value_date, 'debit_amount' => $transaction->debit_amount, 'credit_acct_no' => $transaction->credit_acct_no, 'bif_rcv_acct' => $transaction->bif_rcv_acct, 'bif_rcv_name' => $transaction->bif_rcv_name, 'credit_value_date' => $transaction->credit_value_date, 'credit_amount' => $transaction->credit_amount, 'at_unique_id' => $transaction->at_unique_id, 'bif_ref_no' => $transaction->bif_ref_no, 'atm_order_id' => $transaction->atm_order_id, 'recipt_no' => $transaction->recipt_no, 'api_iss_acct' => $transaction->api_iss_acct, 'api_benff_acct' => $transaction->api_benff_acct, 'authoriser' => $transaction->authoriser, 'remarks' => $transaction->remarks, 'payment_details' => $transaction->payment_details, 'ref_no' => $transaction->ref_no, 'merchant_id' => $transaction->merchant_id, 'term_id' => $transaction->term_id, 'closing_balance' => $runningBalance ]; } /** * Format datetime string * Memformat string datetime */ private function formatDateTime(?string $datetime): string { if (!$datetime) { return ''; } try { return Carbon::createFromFormat('ymdHi', $datetime)->format('d/m/Y H:i'); } catch (Exception $e) { Log::warning('Error formatting datetime', [ 'datetime' => $datetime, 'error' => $e->getMessage() ]); return $datetime; } } /** * Export report data to CSV file * Export data laporan ke file CSV */ private function exportToCsv(array $reportData): string { Log::info('Starting CSV export for closing balance report', [ 'account_number' => $this->accountNumber, 'period' => $this->period, 'record_count' => count($reportData) ]); // Create directory structure $basePath = "closing_balance_reports"; $accountPath = "{$basePath}/{$this->accountNumber}"; Storage::disk($this->disk)->makeDirectory($basePath); Storage::disk($this->disk)->makeDirectory($accountPath); // Generate filename $fileName = "closing_balance_{$this->accountNumber}_{$this->period}.csv"; $filePath = "{$accountPath}/{$fileName}"; // Delete existing file if exists if (Storage::disk($this->disk)->exists($filePath)) { Storage::disk($this->disk)->delete($filePath); } // Create CSV header $csvHeader = [ 'NO', 'TRANS_REFERENCE', 'BOOKING_DATE', 'TRANSACTION_DATE', 'AMOUNT_LCY', 'DEBIT_ACCT_NO', 'DEBIT_VALUE_DATE', 'DEBIT_AMOUNT', 'CREDIT_ACCT_NO', 'BIF_RCV_ACCT', 'BIF_RCV_NAME', 'CREDIT_VALUE_DATE', 'CREDIT_AMOUNT', 'AT_UNIQUE_ID', 'BIF_REF_NO', 'ATM_ORDER_ID', 'RECIPT_NO', 'API_ISS_ACCT', 'API_BENFF_ACCT', 'AUTHORISER', 'REMARKS', 'PAYMENT_DETAILS', 'REF_NO', 'MERCHANT_ID', 'TERM_ID', 'CLOSING_BALANCE' ]; $csvContent = implode('|', $csvHeader) . "\n"; // Add data rows foreach ($reportData as $row) { $csvRow = [ $row['sequence_no'], $row['trans_reference'] ?? '', $row['booking_date'] ?? '', $row['transaction_date'] ?? '', $row['amount_lcy'] ?? '', $row['debit_acct_no'] ?? '', $row['debit_value_date'] ?? '', $row['debit_amount'] ?? '', $row['credit_acct_no'] ?? '', $row['bif_rcv_acct'] ?? '', $row['bif_rcv_name'] ?? '', $row['credit_value_date'] ?? '', $row['credit_amount'] ?? '', $row['at_unique_id'] ?? '', $row['bif_ref_no'] ?? '', $row['atm_order_id'] ?? '', $row['recipt_no'] ?? '', $row['api_iss_acct'] ?? '', $row['api_benff_acct'] ?? '', $row['authoriser'] ?? '', $row['remarks'] ?? '', $row['payment_details'] ?? '', $row['ref_no'] ?? '', $row['merchant_id'] ?? '', $row['term_id'] ?? '', $row['closing_balance'] ?? '' ]; $csvContent .= implode('|', $csvRow) . "\n"; } // Save file Storage::disk($this->disk)->put($filePath, $csvContent); // Verify file creation if (!Storage::disk($this->disk)->exists($filePath)) { throw new Exception("Failed to create CSV file: {$filePath}"); } Log::info('CSV export completed successfully', [ 'file_path' => $filePath, 'file_size' => Storage::disk($this->disk)->size($filePath) ]); return $filePath; } }