webstatement/app/Jobs/GenerateClosingBalanceReportJob.php

<?php

    namespace Modules\Webstatement\Jobs;

    use Carbon\Carbon;
    use Exception;
    use Illuminate\Bus\Queueable;
    use Illuminate\Contracts\Queue\ShouldQueue;
    use Illuminate\Foundation\Bus\Dispatchable;
    use Illuminate\Queue\{InteractsWithQueue, SerializesModels};
    use Illuminate\Support\Facades\{DB, Log, Storage};
    use Modules\Webstatement\Models\{AccountBalance,
        ClosingBalanceReportLog,
        ProcessedClosingBalance,
        StmtEntry,
        StmtEntryDetail};

    /**
     * Job untuk generate laporan closing balance dengan optimasi performa
     * Menggunakan database staging sebelum export CSV
     */
    class GenerateClosingBalanceReportJob implements ShouldQueue
    {
        use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;

        protected $accountNumber;
        protected $period;
        protected $reportLogId;
        protected $groupName;
        protected $chunkSize = 1000;
        protected $disk      = 'local';

        /**
         * Create a new job instance.
         */
        public function __construct(string $accountNumber, string $period, int $reportLogId, string $groupName = 'DEFAULT')
        {
            $this->accountNumber = $accountNumber;
            $this->period        = $period;
            $this->reportLogId   = $reportLogId;
            $this->groupName     = $groupName ?? 'DEFAULT';
        }

        /**
         * Execute the job dengan optimasi performa
         */
        public function handle(): void
        {
            $reportLog = ClosingBalanceReportLog::find($this->reportLogId);

            if (!$reportLog) {
                Log::error('Closing balance report log not found', ['id' => $this->reportLogId]);
                return;
            }

            try {
                Log::info('Starting optimized closing balance report generation', [
                    'account_number' => $this->accountNumber,
                    'period'         => $this->period,
                    'group_name'     => $this->groupName,
                    'report_log_id'  => $this->reportLogId
                ]);

                // Update status to processing
                $reportLog->update([
                    'status'     => 'processing',
                    'updated_at' => now()
                ]);

                // Gunakan satu transaksi untuk seluruh proses
                DB::transaction(function () use ($reportLog) {
                    // Step 1: Process and save to database (fast)
                    $this->processAndSaveClosingBalanceData();

                    // Step 2: Export from database to CSV (fast)
                    $filePath = $this->exportFromDatabaseToCsv();

                    // Get record count from database
                    $recordCount = $this->getProcessedRecordCount();

                    // Update report log with success
                    $reportLog->update([
                        'status'       => 'completed',
                        'file_path'    => $filePath,
                        'file_size'    => Storage::disk($this->disk)->size($filePath),
                        'record_count' => $recordCount,
                        'updated_at'   => now()
                    ]);

                    Log::info('Optimized closing balance report generation completed successfully', [
                        'account_number' => $this->accountNumber,
                        'period'         => $this->period,
                        'file_path'      => $filePath,
                        'record_count'   => $recordCount
                    ]);
                });

            } catch (Exception $e) {
                Log::error('Error generating optimized closing balance report', [
                    'account_number' => $this->accountNumber,
                    'period'         => $this->period,
                    'error'          => $e->getMessage(),
                    'trace'          => $e->getTraceAsString()
                ]);

                $reportLog->update([
                    'status'        => 'failed',
                    'error_message' => $e->getMessage(),
                    'updated_at'    => now()
                ]);

                throw $e;
            }
        }

        /**
         * Process and save closing balance data to database dengan proteksi duplikasi
         * Memproses dan menyimpan data closing balance dengan perlindungan terhadap duplikasi
         */
        private function processAndSaveClosingBalanceData(): void
        {
            $criteria = [
                'account_number' => $this->accountNumber,
                'period'         => $this->period,
                'group_name'     => $this->groupName
            ];

            // HAPUS DB::beginTransaction() - sudah ada di handle()

            // Sederhana: hapus data existing terlebih dahulu seperti ExportStatementJob
            $this->deleteExistingProcessedData($criteria);

            // Get opening balance
            $runningBalance = $this->getOpeningBalance();
            $sequenceNo     = 0;

            Log::info('Starting to process closing balance data', [
                'opening_balance' => $runningBalance,
                'criteria'        => $criteria
            ]);

            // Build query yang sederhana tanpa eliminasi duplicate rumit
            $query = $this->buildTransactionQuery();

            // Proses dan insert data dengan batch updateOrCreate untuk efisiensi
            $query->chunk($this->chunkSize, function ($transactions) use (&$runningBalance, &$sequenceNo) {
                $processedData = $this->prepareProcessedClosingBalanceData($transactions, $runningBalance, $sequenceNo);

                if (!empty($processedData)) {
                    // Gunakan batch processing untuk updateOrCreate
                    $this->batchUpdateOrCreate($processedData);
                }
            });

            // HAPUS DB::commit() - akan di-handle di handle()

            $recordCount = $this->getProcessedRecordCount();
            Log::info('Closing balance data processing completed successfully', [
                'final_sequence' => $sequenceNo,
                'final_balance'  => $runningBalance,
                'record_count'   => $recordCount
            ]);
        }

        /**
         * Get opening balance from account balance table
         * Mengambil saldo awal dari tabel account balance
         */
        private function getOpeningBalance()
        : float
        {
            Log::info('Getting opening balance', [
                'account_number' => $this->accountNumber,
                'period'         => $this->period
            ]);

            // Get previous period based on current period
            $previousPeriod = $this->period === '20250512'
                ? Carbon::createFromFormat('Ymd', $this->period)->subDays(2)->format('Ymd')
                : Carbon::createFromFormat('Ymd', $this->period)->subDay()->format('Ymd');

            $accountBalance = AccountBalance::where('account_number', $this->accountNumber)
                                            ->where('period', $previousPeriod)
                                            ->first();

            if (!$accountBalance) {
                Log::warning('Account balance not found, using 0 as opening balance', [
                    'account_number' => $this->accountNumber,
                    'period'         => $this->period
                ]);
                return 0.0;
            }

            $openingBalance = (float) $accountBalance->actual_balance;

            Log::info('Opening balance retrieved', [
                'account_number'  => $this->accountNumber,
                'opening_balance' => $openingBalance
            ]);

            return $openingBalance;
        }

        /**
         * Build transaction query dengan pendekatan sederhana tanpa eliminasi duplicate rumit
         */
        private function buildTransactionQuery()
        {
            Log::info('Building transaction query', [
                'group_name'     => $this->groupName,
                'account_number' => $this->accountNumber,
                'period'         => $this->period
            ]);

            $modelClass = $this->getModelByGroup();

            $query = $modelClass::select([
                'id',
                'trans_reference',
                'booking_date',
                'amount_lcy',
                'date_time'
            ])
            ->with([
                'ft' => function ($query) {
                    $query->select('ref_no', 'date_time', 'debit_acct_no', 'debit_value_date',
                        'credit_acct_no', 'bif_rcv_acct', 'bif_rcv_name', 'credit_value_date',
                        'at_unique_id', 'bif_ref_no', 'atm_order_id', 'recipt_no',
                        'api_iss_acct', 'api_benff_acct', 'authoriser', 'remarks',
                        'payment_details', 'ref_no', 'merchant_id', 'term_id');
                },
                'dc' => function ($query) {
                    $query->select('id', 'date_time');
                }
            ])
            ->where('account_number', $this->accountNumber)
            ->where('booking_date', $this->period)
            ->orderBy('booking_date')
            ->orderBy('date_time');

            return $query;
        }

        /**
         * Get model class based on group name
         * Mendapatkan class model berdasarkan group name
         */
        private function getModelByGroup()
        {
            Log::info('Determining model by group', [
                'group_name' => $this->groupName
            ]);

            $model = $this->groupName === 'QRIS' ? StmtEntryDetail::class : StmtEntry::class;

            Log::info('Model determined', [
                'group_name'  => $this->groupName,
                'model_class' => $model
            ]);

            return $model;
        }

        /**
         * Prepare processed closing balance data tanpa validasi duplikasi
         */
        private function prepareProcessedClosingBalanceData($transactions, &$runningBalance, &$sequenceNo)
        : array
        {
            $processedData = [];

            foreach ($transactions as $transaction) {
                $sequenceNo++;

                // Process transaction data
                $processedTransactionData = $this->processTransactionData($transaction);

                // Update running balance
                $amount         = (float) $transaction->amount_lcy;
                $runningBalance += $amount;

                // Format transaction date
                $transactionDate = $this->formatDateTime($processedTransactionData['date_time']);

                // Prepare data untuk database insert tanpa unique_hash
                $processedData[] = [
                    'account_number'    => $this->accountNumber,
                    'period'            => $this->period,
                    'group_name'        => $this->groupName,
                    'sequence_no'       => $sequenceNo,
                    'trans_reference'   => $processedTransactionData['trans_reference'],
                    'booking_date'      => $processedTransactionData['booking_date'],
                    'transaction_date'  => $transactionDate,
                    'amount_lcy'        => $processedTransactionData['amount_lcy'],
                    'debit_acct_no'     => $processedTransactionData['debit_acct_no'],
                    'debit_value_date'  => $processedTransactionData['debit_value_date'],
                    'debit_amount'      => $processedTransactionData['debit_amount'],
                    'credit_acct_no'    => $processedTransactionData['credit_acct_no'],
                    'bif_rcv_acct'      => $processedTransactionData['bif_rcv_acct'],
                    'bif_rcv_name'      => $processedTransactionData['bif_rcv_name'],
                    'credit_value_date' => $processedTransactionData['credit_value_date'],
                    'credit_amount'     => $processedTransactionData['credit_amount'],
                    'at_unique_id'      => $processedTransactionData['at_unique_id'],
                    'bif_ref_no'        => $processedTransactionData['bif_ref_no'],
                    'atm_order_id'      => $processedTransactionData['atm_order_id'],
                    'recipt_no'         => $processedTransactionData['recipt_no'],
                    'api_iss_acct'      => $processedTransactionData['api_iss_acct'],
                    'api_benff_acct'    => $processedTransactionData['api_benff_acct'],
                    'authoriser'        => $processedTransactionData['authoriser'],
                    'remarks'           => $processedTransactionData['remarks'],
                    'payment_details'   => $processedTransactionData['payment_details'],
                    'ref_no'            => $processedTransactionData['ref_no'],
                    'merchant_id'       => $processedTransactionData['merchant_id'],
                    'term_id'           => $processedTransactionData['term_id'],
                    'closing_balance'   => $runningBalance,
                    'created_at'        => now(),
                    'updated_at'        => now(),
                ];
            }

            Log::info('Processed closing balance data prepared', [
                'total_records' => count($processedData)
            ]);

            return $processedData;
        }

        /**
         * Process transaction data from ORM result
         * Memproses data transaksi dari hasil ORM
         */
        private function processTransactionData($transaction)
        : array
        {
            Log::info('Processing transaction data', [
                'trans_reference' => $transaction->trans_reference,
                'has_ft_relation' => !is_null($transaction->ft),
                'has_dc_relation' => !is_null($transaction->dc)
            ]);

            // Hitung debit dan credit amount
            $debitAmount  = $transaction->amount_lcy < 0 ? abs($transaction->amount_lcy) : null;
            $creditAmount = $transaction->amount_lcy > 0 ? $transaction->amount_lcy : null;

            // Ambil date_time dari prioritas: ft -> dc -> stmt
            $dateTime = $transaction->ft?->date_time ??
                $transaction->dc?->date_time ??
                $transaction->date_time;

            $processedData = [
                'trans_reference'   => $transaction->trans_reference,
                'booking_date'      => $transaction->booking_date,
                'amount_lcy'        => $transaction->amount_lcy,
                'debit_amount'      => $debitAmount,
                'credit_amount'     => $creditAmount,
                'date_time'         => $dateTime,
                // Data dari TempFundsTransfer melalui relasi
                'debit_acct_no'     => $transaction->ft?->debit_acct_no,
                'debit_value_date'  => $transaction->ft?->debit_value_date,
                'credit_acct_no'    => $transaction->ft?->credit_acct_no,
                'bif_rcv_acct'      => $transaction->ft?->bif_rcv_acct,
                'bif_rcv_name'      => $transaction->ft?->bif_rcv_name,
                'credit_value_date' => $transaction->ft?->credit_value_date,
                'at_unique_id'      => $transaction->ft?->at_unique_id,
                'bif_ref_no'        => $transaction->ft?->bif_ref_no,
                'atm_order_id'      => $transaction->ft?->atm_order_id,
                'recipt_no'         => $transaction->ft?->recipt_no,
                'api_iss_acct'      => $transaction->ft?->api_iss_acct,
                'api_benff_acct'    => $transaction->ft?->api_benff_acct,
                'authoriser'        => $transaction->ft?->authoriser,
                'remarks'           => $transaction->ft?->remarks,
                'payment_details'   => $transaction->ft?->payment_details,
                'ref_no'            => $transaction->ft?->ref_no,
                'merchant_id'       => $transaction->ft?->merchant_id,
                'term_id'           => $transaction->ft?->term_id,
            ];

            Log::info('Transaction data processed successfully', [
                'trans_reference' => $transaction->trans_reference,
                'final_date_time' => $dateTime,
                'debit_amount'    => $debitAmount,
                'credit_amount'   => $creditAmount
            ]);

            return $processedData;
        }

        /**
         * Format datetime string
         * Memformat string datetime
         */
        private function formatDateTime(?string $datetime)
        : string
        {
            if (!$datetime) {
                return '';
            }

            try {
                return Carbon::createFromFormat('ymdHi', $datetime)->format('d/m/Y H:i');
            } catch (Exception $e) {
                Log::warning('Error formatting datetime', [
                    'datetime' => $datetime,
                    'error'    => $e->getMessage()
                ]);
                return $datetime;
            }
        }
        /**
         * Export from database to CSV (very fast)
         */
        private function exportFromDatabaseToCsv()
        : string
        {
            Log::info('Starting CSV export from database for closing balance report', [
                'account_number' => $this->accountNumber,
                'period'         => $this->period,
                'group_name'     => $this->groupName
            ]);

            // Create directory structure
            $basePath    = "closing_balance_reports";
            $accountPath = "{$basePath}/{$this->accountNumber}";

            Storage::disk($this->disk)->makeDirectory($basePath);
            Storage::disk($this->disk)->makeDirectory($accountPath);

            // Generate filename
            $fileName = "closing_balance_{$this->accountNumber}_{$this->period}_{$this->groupName}.csv";
            $filePath = "{$accountPath}/{$fileName}";

            // Delete existing file if exists
            if (Storage::disk($this->disk)->exists($filePath)) {
                Storage::disk($this->disk)->delete($filePath);
            }

            // Create CSV header
            $csvHeader = [
                'NO',
                'TRANS_REFERENCE',
                'BOOKING_DATE',
                'TRANSACTION_DATE',
                'AMOUNT_LCY',
                'DEBIT_ACCT_NO',
                'DEBIT_VALUE_DATE',
                'DEBIT_AMOUNT',
                'CREDIT_ACCT_NO',
                'BIF_RCV_ACCT',
                'BIF_RCV_NAME',
                'CREDIT_VALUE_DATE',
                'CREDIT_AMOUNT',
                'AT_UNIQUE_ID',
                'BIF_REF_NO',
                'ATM_ORDER_ID',
                'RECIPT_NO',
                'API_ISS_ACCT',
                'API_BENFF_ACCT',
                'AUTHORISER',
                'REMARKS',
                'PAYMENT_DETAILS',
                'REF_NO',
                'MERCHANT_ID',
                'TERM_ID',
                'CLOSING_BALANCE'
            ];

            $csvContent = implode('|', $csvHeader) . "\n";
            Storage::disk($this->disk)->put($filePath, $csvContent);

            // Inisialisasi counter untuk sequence number
            $sequenceCounter = 1;
            $processedHashes = [];

            ProcessedClosingBalance::where('account_number', $this->accountNumber)
                                   ->where('period', $this->period)
                                   ->where('group_name', $this->groupName)
                                   ->orderBy('sequence_no')
                                   ->chunk($this->chunkSize, function ($records) use ($filePath, &$sequenceCounter, &$processedHashes) {
                                    $csvContent = [];
                                       foreach ($records as $record) {
                                           // Pengecekan unique_hash: skip jika sudah diproses
                                           if (in_array($record->unique_hash, $processedHashes)) {
                                               Log::debug('Skipping duplicate unique_hash in CSV export', [
                                                   'unique_hash'     => $record->unique_hash,
                                                   'trans_reference' => $record->trans_reference
                                               ]);
                                               continue;
                                           }

                                           // Tandai unique_hash sebagai sudah diproses
                                           $processedHashes[] = $record->unique_hash;

                                           $csvRow = [
                                               $sequenceCounter++,
                                               // Gunakan counter yang bertambah, bukan sequence_no dari database
                                               $record->trans_reference ?? '',
                                               $record->booking_date ?? '',
                                               $record->transaction_date ?? '',
                                               $record->amount_lcy ?? '',
                                               $record->debit_acct_no ?? '',
                                               $record->debit_value_date ?? '',
                                               $record->debit_amount ?? '',
                                               $record->credit_acct_no ?? '',
                                               $record->bif_rcv_acct ?? '',
                                               $record->bif_rcv_name ?? '',
                                               $record->credit_value_date ?? '',
                                               $record->credit_amount ?? '',
                                               $record->at_unique_id ?? '',
                                               $record->bif_ref_no ?? '',
                                               $record->atm_order_id ?? '',
                                               $record->recipt_no ?? '',
                                               $record->api_iss_acct ?? '',
                                               $record->api_benff_acct ?? '',
                                               $record->authoriser ?? '',
                                               $record->remarks ?? '',
                                               $record->payment_details ?? '',
                                               $record->ref_no ?? '',
                                               $record->merchant_id ?? '',
                                               $record->term_id ?? '',
                                               $record->closing_balance ?? ''
                                           ];

                                           $csvContent .= implode('|', $csvRow) . "\n";
                                       }

                                       if (!empty($csvContent)) {
                                           Storage::disk($this->disk)->append($filePath, $csvContent);

                                           Log::debug('CSV content appended', [
                                               'records_processed' => substr_count($csvContent, "\n"),
                                               'current_sequence'  => $sequenceCounter - 1
                                           ]);
                                       }
                                   });

            // Verify file creation
            if (!Storage::disk($this->disk)->exists($filePath)) {
                throw new Exception("Failed to create CSV file: {$filePath}");
            }

            Log::info('CSV export from database completed successfully', [
                'file_path' => $filePath,
                'file_size' => Storage::disk($this->disk)->size($filePath)
            ]);

            return $filePath;
        }

        /**
         * Get processed record count
         */
        private function getProcessedRecordCount()
        : int
        {
            return ProcessedClosingBalance::where('account_number', $this->accountNumber)
                                          ->where('period', $this->period)
                                          ->where('group_name', $this->groupName)
                                          ->count();
        }

        /**
         * Delete existing processed data dengan pendekatan sederhana seperti ExportStatementJob
         */
        private function deleteExistingProcessedData(array $criteria)
        : void
        {
            Log::info('Deleting existing processed data', $criteria);

            $deletedCount = ProcessedClosingBalance::where('account_number', $criteria['account_number'])
                                                   ->where('period', $criteria['period'])
                                                   ->delete();

            Log::info('Existing processed data deleted', [
                'deleted_count' => $deletedCount,
                'criteria'      => $criteria
            ]);
        }

        /**
         * Batch update or create untuk mengurangi jumlah query dan lock
         * Menggunakan pendekatan yang lebih efisien untuk menghindari max_lock_per_transaction
         */
        private function batchUpdateOrCreate(array $processedData): void
        {
            Log::info('Starting batch updateOrCreate', [
                'batch_size' => count($processedData)
            ]);

            // Kumpulkan semua trans_reference yang akan diproses
            $transReferences = collect($processedData)->pluck('trans_reference')->toArray();

            // Ambil data yang sudah ada dalam satu query
            $existingRecords = ProcessedClosingBalance::where('account_number', $this->accountNumber)
                ->where('period', $this->period)
                ->where('group_name', $this->groupName)
                ->whereIn('trans_reference', $transReferences)
                ->get()
                ->keyBy(function ($item) {
                    return $item->trans_reference . '_' . $item->amount_lcy;
                });

            $toInsert = [];
            $toUpdate = [];

            foreach ($processedData as $data) {
                $key = $data['trans_reference'] . '_' . $data['amount_lcy'];

                if ($existingRecords->has($key)) {
                    // Record sudah ada, siapkan untuk update
                    $existingRecord = $existingRecords->get($key);
                    $toUpdate[] = [
                        'id' => $existingRecord->id,
                        'data' => $data
                    ];
                } else {
                    // Record baru, siapkan untuk insert
                    $toInsert[] = $data;
                }
            }

            // Batch insert untuk record baru
            if (!empty($toInsert)) {
                DB::table('processed_closing_balances')->insert($toInsert);
                Log::info('Batch insert completed', ['count' => count($toInsert)]);
            }

            // Batch update untuk record yang sudah ada
            if (!empty($toUpdate)) {
                foreach ($toUpdate as $updateItem) {
                    ProcessedClosingBalance::where('id', $updateItem['id'])
                        ->update($updateItem['data']);
                }
                Log::info('Batch update completed', ['count' => count($toUpdate)]);
            }

            Log::info('Batch updateOrCreate completed successfully', [
                'inserted' => count($toInsert),
                'updated' => count($toUpdate)
            ]);
        }
    }