webstatement/app/Jobs/GenerateClosingBalanceReportJob.php

<?php

namespace Modules\Webstatement\Jobs;

use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
use Carbon\Carbon;
use Exception;
use Modules\Webstatement\Models\AccountBalance;
use Modules\Webstatement\Models\ClosingBalanceReportLog;
use Modules\Webstatement\Models\ProcessedClosingBalance;
use Modules\Webstatement\Models\StmtEntry;
use Modules\Webstatement\Models\StmtEntryDetail;
use Modules\Webstatement\Models\TempFundsTransfer;
use Modules\Webstatement\Models\DataCapture;

/**
 * Job untuk generate laporan closing balance dengan optimasi performa
 * Menggunakan database staging sebelum export CSV
 */
class GenerateClosingBalanceReportJob implements ShouldQueue
{
    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;

    protected $accountNumber;
    protected $period;
    protected $reportLogId;
    protected $groupName;
    protected $chunkSize = 1000;
    protected $disk = 'local';

    /**
     * Create a new job instance.
     */
    public function __construct(string $accountNumber, string $period, int $reportLogId, string $groupName='DEFAULT')
    {
        $this->accountNumber = $accountNumber;
        $this->period = $period;
        $this->reportLogId = $reportLogId;
        $this->groupName = $groupName ?? 'DEFAULT';
    }

    /**
     * Execute the job dengan optimasi performa
     */
    public function handle(): void
    {
        $reportLog = ClosingBalanceReportLog::find($this->reportLogId);

        if (!$reportLog) {
            Log::error('Closing balance report log not found', ['id' => $this->reportLogId]);
            return;
        }

        try {
            Log::info('Starting optimized closing balance report generation', [
                'account_number' => $this->accountNumber,
                'period' => $this->period,
                'group_name' => $this->groupName,
                'report_log_id' => $this->reportLogId
            ]);

            DB::beginTransaction();

            // Update status to processing
            $reportLog->update([
                'status' => 'processing',
                'updated_at' => now()
            ]);

            // Step 1: Process and save to database (fast)
            $this->processAndSaveClosingBalanceData();

            // Step 2: Export from database to CSV (fast)
            $filePath = $this->exportFromDatabaseToCsv();

            // Get record count from database
            $recordCount = $this->getProcessedRecordCount();

            // Update report log with success
            $reportLog->update([
                'status' => 'completed',
                'file_path' => $filePath,
                'file_size' => Storage::disk($this->disk)->size($filePath),
                'record_count' => $recordCount,
                'updated_at' => now()
            ]);

            DB::commit();

            Log::info('Optimized closing balance report generation completed successfully', [
                'account_number' => $this->accountNumber,
                'period' => $this->period,
                'file_path' => $filePath,
                'record_count' => $recordCount
            ]);

        } catch (Exception $e) {
            DB::rollback();

            Log::error('Error generating optimized closing balance report', [
                'account_number' => $this->accountNumber,
                'period' => $this->period,
                'error' => $e->getMessage(),
                'trace' => $e->getTraceAsString()
            ]);

            $reportLog->update([
                'status' => 'failed',
                'error_message' => $e->getMessage(),
                'updated_at' => now()
            ]);

            throw $e;
        }
    }

    /**
     * Process and save closing balance data to database
     */
    private function processAndSaveClosingBalanceData(): void
    {
        $criteria = [
            'account_number' => $this->accountNumber,
            'period' => $this->period,
            'group_name' => $this->groupName
        ];

        // Delete existing processed data
        $this->deleteExistingProcessedData($criteria);

        // Get opening balance
        $runningBalance = $this->getOpeningBalance();
        $sequenceNo = 0;

        Log::info('Starting to process and save closing balance data', [
            'opening_balance' => $runningBalance,
            'criteria' => $criteria
        ]);

        // Build query
        $query = $this->buildTransactionQuery();

        // Process in chunks and save to database
        $query->chunk($this->chunkSize, function ($transactions) use (&$runningBalance, &$sequenceNo) {
            $processedData = $this->prepareProcessedClosingBalanceData($transactions, $runningBalance, $sequenceNo);

            if (!empty($processedData)) {
                DB::table('processed_closing_balances')->insert($processedData);
            }

            Log::info('Chunk processed and saved to database', [
                'chunk_size' => count($processedData),
                'current_balance' => $runningBalance
            ]);
        });

        Log::info('Closing balance data processing completed', [
            'final_sequence' => $sequenceNo,
            'final_balance' => $runningBalance
        ]);
    }

    /**
     * Delete existing processed data
     */
    private function deleteExistingProcessedData(array $criteria): void
    {
        ProcessedClosingBalance::where('account_number', $criteria['account_number'])
            ->where('period', $criteria['period'])
            ->where('group_name', $criteria['group_name'])
            ->delete();
    }

    /**
     * Prepare processed closing balance data for batch insert
     */
    private function prepareProcessedClosingBalanceData($transactions, &$runningBalance, &$sequenceNo): array
    {
        $processedData = [];

        foreach ($transactions as $transaction) {
            $sequenceNo++;

            // Process transaction data
            $processedTransactionData = $this->processTransactionData($transaction);

            // Update running balance
            $amount = (float) $transaction->amount_lcy;
            $runningBalance += $amount;

            // Format transaction date
            $transactionDate = $this->formatDateTime($processedTransactionData['date_time']);

            // Prepare data for database insert
            $processedData[] = [
                'account_number' => $this->accountNumber,
                'period' => $this->period,
                'group_name' => $this->groupName,
                'sequence_no' => $sequenceNo,
                'trans_reference' => $processedTransactionData['trans_reference'],
                'booking_date' => $processedTransactionData['booking_date'],
                'transaction_date' => $transactionDate,
                'amount_lcy' => $processedTransactionData['amount_lcy'],
                'debit_acct_no' => $processedTransactionData['debit_acct_no'],
                'debit_value_date' => $processedTransactionData['debit_value_date'],
                'debit_amount' => $processedTransactionData['debit_amount'],
                'credit_acct_no' => $processedTransactionData['credit_acct_no'],
                'bif_rcv_acct' => $processedTransactionData['bif_rcv_acct'],
                'bif_rcv_name' => $processedTransactionData['bif_rcv_name'],
                'credit_value_date' => $processedTransactionData['credit_value_date'],
                'credit_amount' => $processedTransactionData['credit_amount'],
                'at_unique_id' => $processedTransactionData['at_unique_id'],
                'bif_ref_no' => $processedTransactionData['bif_ref_no'],
                'atm_order_id' => $processedTransactionData['atm_order_id'],
                'recipt_no' => $processedTransactionData['recipt_no'],
                'api_iss_acct' => $processedTransactionData['api_iss_acct'],
                'api_benff_acct' => $processedTransactionData['api_benff_acct'],
                'authoriser' => $processedTransactionData['authoriser'],
                'remarks' => $processedTransactionData['remarks'],
                'payment_details' => $processedTransactionData['payment_details'],
                'ref_no' => $processedTransactionData['ref_no'],
                'merchant_id' => $processedTransactionData['merchant_id'],
                'term_id' => $processedTransactionData['term_id'],
                'closing_balance' => $runningBalance,
                'created_at' => now(),
                'updated_at' => now()
            ];
        }

        return $processedData;
    }

    /**
     * Export from database to CSV (very fast)
     */
    private function exportFromDatabaseToCsv(): string
    {
        Log::info('Starting CSV export from database for closing balance report', [
            'account_number' => $this->accountNumber,
            'period' => $this->period,
            'group_name' => $this->groupName
        ]);

        // Create directory structure
        $basePath = "closing_balance_reports";
        $accountPath = "{$basePath}/{$this->accountNumber}";

        Storage::disk($this->disk)->makeDirectory($basePath);
        Storage::disk($this->disk)->makeDirectory($accountPath);

        // Generate filename
        $fileName = "closing_balance_{$this->accountNumber}_{$this->period}_{$this->groupName}.csv";
        $filePath = "{$accountPath}/{$fileName}";

        // Delete existing file if exists
        if (Storage::disk($this->disk)->exists($filePath)) {
            Storage::disk($this->disk)->delete($filePath);
        }

        // Create CSV header
        $csvHeader = [
            'NO',
            'TRANS_REFERENCE',
            'BOOKING_DATE',
            'TRANSACTION_DATE',
            'AMOUNT_LCY',
            'DEBIT_ACCT_NO',
            'DEBIT_VALUE_DATE',
            'DEBIT_AMOUNT',
            'CREDIT_ACCT_NO',
            'BIF_RCV_ACCT',
            'BIF_RCV_NAME',
            'CREDIT_VALUE_DATE',
            'CREDIT_AMOUNT',
            'AT_UNIQUE_ID',
            'BIF_REF_NO',
            'ATM_ORDER_ID',
            'RECIPT_NO',
            'API_ISS_ACCT',
            'API_BENFF_ACCT',
            'AUTHORISER',
            'REMARKS',
            'PAYMENT_DETAILS',
            'REF_NO',
            'MERCHANT_ID',
            'TERM_ID',
            'CLOSING_BALANCE'
        ];

        $csvContent = implode('|', $csvHeader) . "\n";
        Storage::disk($this->disk)->put($filePath, $csvContent);

        // Export data from database in chunks
        ProcessedClosingBalance::where('account_number', $this->accountNumber)
            ->where('period', $this->period)
            ->where('group_name', $this->groupName)
            ->orderBy('sequence_no')
            ->chunk($this->chunkSize, function ($records) use ($filePath) {
                $csvContent = '';
                foreach ($records as $record) {
                    $csvRow = [
                        $record->sequence_no,
                        $record->trans_reference ?? '',
                        $record->booking_date ?? '',
                        $record->transaction_date ?? '',
                        $record->amount_lcy ?? '',
                        $record->debit_acct_no ?? '',
                        $record->debit_value_date ?? '',
                        $record->debit_amount ?? '',
                        $record->credit_acct_no ?? '',
                        $record->bif_rcv_acct ?? '',
                        $record->bif_rcv_name ?? '',
                        $record->credit_value_date ?? '',
                        $record->credit_amount ?? '',
                        $record->at_unique_id ?? '',
                        $record->bif_ref_no ?? '',
                        $record->atm_order_id ?? '',
                        $record->recipt_no ?? '',
                        $record->api_iss_acct ?? '',
                        $record->api_benff_acct ?? '',
                        $record->authoriser ?? '',
                        $record->remarks ?? '',
                        $record->payment_details ?? '',
                        $record->ref_no ?? '',
                        $record->merchant_id ?? '',
                        $record->term_id ?? '',
                        $record->closing_balance ?? ''
                    ];

                    $csvContent .= implode('|', $csvRow) . "\n";
                }

                if (!empty($csvContent)) {
                    Storage::disk($this->disk)->append($filePath, $csvContent);
                }
            });

        // Verify file creation
        if (!Storage::disk($this->disk)->exists($filePath)) {
            throw new Exception("Failed to create CSV file: {$filePath}");
        }

        Log::info('CSV export from database completed successfully', [
            'file_path' => $filePath,
            'file_size' => Storage::disk($this->disk)->size($filePath)
        ]);

        return $filePath;
    }

    /**
     * Get processed record count
     */
    private function getProcessedRecordCount(): int
    {
        return ProcessedClosingBalance::where('account_number', $this->accountNumber)
            ->where('period', $this->period)
            ->where('group_name', $this->groupName)
            ->count();
    }

    /**
     * Get opening balance from account balance table
     * Mengambil saldo awal dari tabel account balance
     */
    private function getOpeningBalance(): float
    {
        Log::info('Getting opening balance', [
            'account_number' => $this->accountNumber,
            'period' => $this->period
        ]);

        // Get previous period based on current period
        $previousPeriod = $this->period === '20250512'
            ? Carbon::createFromFormat('Ymd', $this->period)->subDays(2)->format('Ymd')
            : Carbon::createFromFormat('Ymd', $this->period)->subDay()->format('Ymd');

        $accountBalance = AccountBalance::where('account_number', $this->accountNumber)
            ->where('period', $previousPeriod)
            ->first();

        if (!$accountBalance) {
            Log::warning('Account balance not found, using 0 as opening balance', [
                'account_number' => $this->accountNumber,
                'period' => $this->period
            ]);
            return 0.0;
        }

        $openingBalance = (float) $accountBalance->actual_balance;

        Log::info('Opening balance retrieved', [
            'account_number' => $this->accountNumber,
            'opening_balance' => $openingBalance
        ]);

        return $openingBalance;
    }

    /**
     * Build transaction query using pure Eloquent relationships
     * Membangun query transaksi menggunakan relasi Eloquent murni
     */
    private function buildTransactionQuery()
    {
        Log::info('Building transaction query using pure Eloquent relationships', [
            'group_name' => $this->groupName,
            'account_number' => $this->accountNumber,
            'period' => $this->period
        ]);

        // Tentukan model berdasarkan group name
        $modelClass = $this->getModelByGroup();

        // Build query menggunakan pure Eloquent dengan distinct pada kombinasi trans_reference dan amount_lcy
        $query = $modelClass::select([
                'id',
                'trans_reference',
                'booking_date',
                'amount_lcy',
                'date_time'
            ])
            ->where('account_number', $this->accountNumber)
            ->where('booking_date', $this->period)
            ->distinct(['trans_reference', 'amount_lcy']) // Distinct pada kombinasi trans_reference dan amount_lcy
            ->orderBy('trans_reference')
            ->orderBy('amount_lcy')
            ->orderBy('booking_date')
            ->orderBy('date_time');

        Log::info('Transaction query built successfully using pure Eloquent with distinct trans_reference and amount_lcy', [
            'model_class' => $modelClass,
            'account_number' => $this->accountNumber,
            'period' => $this->period
        ]);

        return $query;
    }

    /**
     * Get model class based on group name
     * Mendapatkan class model berdasarkan group name
     */
    private function getModelByGroup()
    {
        Log::info('Determining model by group', [
            'group_name' => $this->groupName
        ]);

        $model = $this->groupName === 'QRIS' ? StmtEntryDetail::class : StmtEntry::class;

        Log::info('Model determined', [
            'group_name' => $this->groupName,
            'model_class' => $model
        ]);

        return $model;
    }

    /**
     * Process transaction data from ORM result
     * Memproses data transaksi dari hasil ORM
     */
    private function processTransactionData($transaction): array
    {
        Log::info('Processing transaction data', [
            'trans_reference' => $transaction->trans_reference,
            'has_ft_relation' => !is_null($transaction->ft),
            'has_dc_relation' => !is_null($transaction->dc)
        ]);

        // Hitung debit dan credit amount
        $debitAmount = $transaction->amount_lcy < 0 ? abs($transaction->amount_lcy) : null;
        $creditAmount = $transaction->amount_lcy > 0 ? $transaction->amount_lcy : null;

        // Ambil date_time dari prioritas: ft -> dc -> stmt
        $dateTime = $transaction->ft?->date_time ??
                   $transaction->dc?->date_time ??
                   $transaction->date_time;

        $processedData = [
            'trans_reference' => $transaction->trans_reference,
            'booking_date' => $transaction->booking_date,
            'amount_lcy' => $transaction->amount_lcy,
            'debit_amount' => $debitAmount,
            'credit_amount' => $creditAmount,
            'date_time' => $dateTime,
            // Data dari TempFundsTransfer melalui relasi
            'debit_acct_no' => $transaction->ft?->debit_acct_no,
            'debit_value_date' => $transaction->ft?->debit_value_date,
            'credit_acct_no' => $transaction->ft?->credit_acct_no,
            'bif_rcv_acct' => $transaction->ft?->bif_rcv_acct,
            'bif_rcv_name' => $transaction->ft?->bif_rcv_name,
            'credit_value_date' => $transaction->ft?->credit_value_date,
            'at_unique_id' => $transaction->ft?->at_unique_id,
            'bif_ref_no' => $transaction->ft?->bif_ref_no,
            'atm_order_id' => $transaction->ft?->atm_order_id,
            'recipt_no' => $transaction->ft?->recipt_no,
            'api_iss_acct' => $transaction->ft?->api_iss_acct,
            'api_benff_acct' => $transaction->ft?->api_benff_acct,
            'authoriser' => $transaction->ft?->authoriser,
            'remarks' => $transaction->ft?->remarks,
            'payment_details' => $transaction->ft?->payment_details,
            'ref_no' => $transaction->ft?->ref_no,
            'merchant_id' => $transaction->ft?->merchant_id,
            'term_id' => $transaction->ft?->term_id,
        ];

        Log::info('Transaction data processed successfully', [
            'trans_reference' => $transaction->trans_reference,
            'final_date_time' => $dateTime,
            'debit_amount' => $debitAmount,
            'credit_amount' => $creditAmount
        ]);

        return $processedData;
    }

    /**
     * Updated generateReportData method using pure ORM
     * Method generateReportData yang diperbarui menggunakan ORM murni
     */
    private function generateReportData(): array
    {
        Log::info('Starting report data generation using pure ORM', [
            'account_number' => $this->accountNumber,
            'period' => $this->period,
            'group_name' => $this->groupName,
            'chunk_size' => $this->chunkSize
        ]);

        $reportData = [];
        $runningBalance = $this->getOpeningBalance();
        $sequenceNo = 1;

        try {
            DB::beginTransaction();

            // Build query menggunakan pure ORM
            $query = $this->buildTransactionQuery();

            // Process data dalam chunks untuk efisiensi memory
            $query->chunk($this->chunkSize, function ($transactions) use (&$reportData, &$runningBalance, &$sequenceNo) {
                Log::info('Processing transaction chunk', [
                    'chunk_size' => $transactions->count(),
                    'current_sequence' => $sequenceNo,
                    'current_balance' => $runningBalance
                ]);

                foreach ($transactions as $transaction) {
                    // Process transaction data
                    $processedData = $this->processTransactionData($transaction);

                    // Update running balance
                    $amount = (float) $transaction->amount_lcy;
                    $runningBalance += $amount;

                    // Format transaction date
                    $transactionDate = $this->formatDateTime($processedData['date_time']);

                    // Build report data row
                    $reportData[] = $this->buildReportDataRow(
                        (object) $processedData,
                        $sequenceNo,
                        $transactionDate,
                        $runningBalance
                    );

                    $sequenceNo++;
                }

                Log::info('Chunk processed successfully', [
                    'processed_count' => $transactions->count(),
                    'total_records_so_far' => count($reportData),
                    'current_balance' => $runningBalance
                ]);
            });

            DB::commit();

            Log::info('Report data generation completed using pure ORM', [
                'total_records' => count($reportData),
                'final_balance' => $runningBalance,
                'final_sequence' => $sequenceNo - 1
            ]);

        } catch (Exception $e) {
            DB::rollback();

            Log::error('Error generating report data using pure ORM', [
                'error' => $e->getMessage(),
                'trace' => $e->getTraceAsString(),
                'account_number' => $this->accountNumber,
                'period' => $this->period
            ]);

            throw $e;
        }

        return $reportData;
    }

    /**
     * Build report data row from transaction
     * Membangun baris data laporan dari transaksi
     */
    private function buildReportDataRow($transaction, int $sequenceNo, string $transactionDate, float $runningBalance): array
    {
        return [
            'sequence_no' => $sequenceNo,
            'trans_reference' => $transaction->trans_reference,
            'booking_date' => $transaction->booking_date,
            'transaction_date' => $transactionDate,
            'amount_lcy' => $transaction->amount_lcy,
            'debit_acct_no' => $transaction->debit_acct_no,
            'debit_value_date' => $transaction->debit_value_date,
            'debit_amount' => $transaction->debit_amount,
            'credit_acct_no' => $transaction->credit_acct_no,
            'bif_rcv_acct' => $transaction->bif_rcv_acct,
            'bif_rcv_name' => $transaction->bif_rcv_name,
            'credit_value_date' => $transaction->credit_value_date,
            'credit_amount' => $transaction->credit_amount,
            'at_unique_id' => $transaction->at_unique_id,
            'bif_ref_no' => $transaction->bif_ref_no,
            'atm_order_id' => $transaction->atm_order_id,
            'recipt_no' => $transaction->recipt_no,
            'api_iss_acct' => $transaction->api_iss_acct,
            'api_benff_acct' => $transaction->api_benff_acct,
            'authoriser' => $transaction->authoriser,
            'remarks' => $transaction->remarks,
            'payment_details' => $transaction->payment_details,
            'ref_no' => $transaction->ref_no,
            'merchant_id' => $transaction->merchant_id,
            'term_id' => $transaction->term_id,
            'closing_balance' => $runningBalance
        ];
    }

    /**
     * Format datetime string
     * Memformat string datetime
     */
    private function formatDateTime(?string $datetime): string
    {
        if (!$datetime) {
            return '';
        }

        try {
            return Carbon::createFromFormat('ymdHi', $datetime)->format('d/m/Y H:i');
        } catch (Exception $e) {
            Log::warning('Error formatting datetime', [
                'datetime' => $datetime,
                'error' => $e->getMessage()
            ]);
            return $datetime;
        }
    }

    /**
     * Export report data to CSV file
     * Export data laporan ke file CSV
     */
    private function exportToCsv(array $reportData): string
    {
        Log::info('Starting CSV export for closing balance report', [
            'account_number' => $this->accountNumber,
            'period' => $this->period,
            'record_count' => count($reportData)
        ]);

        // Create directory structure
        $basePath = "closing_balance_reports";
        $accountPath = "{$basePath}/{$this->accountNumber}";

        Storage::disk($this->disk)->makeDirectory($basePath);
        Storage::disk($this->disk)->makeDirectory($accountPath);

        // Generate filename
        $fileName = "closing_balance_{$this->accountNumber}_{$this->period}.csv";
        $filePath = "{$accountPath}/{$fileName}";

        // Delete existing file if exists
        if (Storage::disk($this->disk)->exists($filePath)) {
            Storage::disk($this->disk)->delete($filePath);
        }

        // Create CSV header
        $csvHeader = [
            'NO',
            'TRANS_REFERENCE',
            'BOOKING_DATE',
            'TRANSACTION_DATE',
            'AMOUNT_LCY',
            'DEBIT_ACCT_NO',
            'DEBIT_VALUE_DATE',
            'DEBIT_AMOUNT',
            'CREDIT_ACCT_NO',
            'BIF_RCV_ACCT',
            'BIF_RCV_NAME',
            'CREDIT_VALUE_DATE',
            'CREDIT_AMOUNT',
            'AT_UNIQUE_ID',
            'BIF_REF_NO',
            'ATM_ORDER_ID',
            'RECIPT_NO',
            'API_ISS_ACCT',
            'API_BENFF_ACCT',
            'AUTHORISER',
            'REMARKS',
            'PAYMENT_DETAILS',
            'REF_NO',
            'MERCHANT_ID',
            'TERM_ID',
            'CLOSING_BALANCE'
        ];

        $csvContent = implode('|', $csvHeader) . "\n";

        // Add data rows
        foreach ($reportData as $row) {
            $csvRow = [
                $row['sequence_no'],
                $row['trans_reference'] ?? '',
                $row['booking_date'] ?? '',
                $row['transaction_date'] ?? '',
                $row['amount_lcy'] ?? '',
                $row['debit_acct_no'] ?? '',
                $row['debit_value_date'] ?? '',
                $row['debit_amount'] ?? '',
                $row['credit_acct_no'] ?? '',
                $row['bif_rcv_acct'] ?? '',
                $row['bif_rcv_name'] ?? '',
                $row['credit_value_date'] ?? '',
                $row['credit_amount'] ?? '',
                $row['at_unique_id'] ?? '',
                $row['bif_ref_no'] ?? '',
                $row['atm_order_id'] ?? '',
                $row['recipt_no'] ?? '',
                $row['api_iss_acct'] ?? '',
                $row['api_benff_acct'] ?? '',
                $row['authoriser'] ?? '',
                $row['remarks'] ?? '',
                $row['payment_details'] ?? '',
                $row['ref_no'] ?? '',
                $row['merchant_id'] ?? '',
                $row['term_id'] ?? '',
                $row['closing_balance'] ?? ''
            ];

            $csvContent .= implode('|', $csvRow) . "\n";
        }

        // Save file
        Storage::disk($this->disk)->put($filePath, $csvContent);

        // Verify file creation
        if (!Storage::disk($this->disk)->exists($filePath)) {
            throw new Exception("Failed to create CSV file: {$filePath}");
        }

        Log::info('CSV export completed successfully', [
            'file_path' => $filePath,
            'file_size' => Storage::disk($this->disk)->size($filePath)
        ]);

        return $filePath;
    }
}