webstatement/app/Jobs/ProcessStmtEntryDetailDataJob.php

<?php

namespace Modules\Webstatement\Jobs;

use Exception;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
use Modules\Webstatement\Models\StmtEntryDetail;
use Illuminate\Support\Facades\DB;

class ProcessStmtEntryDetailDataJob implements ShouldQueue
{
    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;

    private const CSV_DELIMITER      = '~';
    private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds
    private const FILENAME           = 'ST.STMT.ENTRY.DETAIL.csv';
    private const DISK_NAME          = 'staging';
    private const CHUNK_SIZE         = 1000; // Process data in chunks to reduce memory usage

    private string $period = '';
    private int    $processedCount = 0;
    private int    $errorCount     = 0;
    private array  $entryBatch     = [];

    /**
     * Create a new job instance.
     *
     * @param string $period Periode data yang akan diproses
     */
    public function __construct(string $period = '')
    {
        $this->period = $period;
        Log::info('ProcessStmtEntryDetailDataJob initialized', ['period' => $period]);
    }

    /**
     * Execute the job.
     *
     * @return void
     * @throws Exception
     */
    public function handle(): void
    {
        try {
            Log::info('Memulai ProcessStmtEntryDetailDataJob', ['period' => $this->period]);

            $this->initializeJob();

            if ($this->period === '') {
                Log::warning('No period provided for statement entry detail data processing');
                return;
            }

            $this->processPeriod();
            $this->logJobCompletion();

            Log::info('ProcessStmtEntryDetailDataJob selesai berhasil');
        } catch (Exception $e) {
            Log::error('Error in ProcessStmtEntryDetailDataJob: ' . $e->getMessage());
            throw $e;
        }
    }

    /**
     * Inisialisasi job dengan pengaturan awal
     *
     * @return void
     */
    private function initializeJob(): void
    {
        set_time_limit(self::MAX_EXECUTION_TIME);
        $this->processedCount = 0;
        $this->errorCount     = 0;
        $this->entryBatch     = [];

        Log::info('Job initialized', [
            'max_execution_time' => self::MAX_EXECUTION_TIME,
            'chunk_size' => self::CHUNK_SIZE
        ]);
    }

    /**
     * Proses data untuk periode tertentu
     *
     * @return void
     */
    private function processPeriod(): void
    {
        $disk     = Storage::disk(self::DISK_NAME);
        $filename = "{$this->period}." . self::FILENAME;
        $filePath = "{$this->period}/$filename";

        Log::info('Memulai proses periode', ['file_path' => $filePath]);

        if (!$this->validateFile($disk, $filePath)) {
            return;
        }

        $tempFilePath = $this->createTemporaryFile($disk, $filePath, $filename);
        $this->processFile($tempFilePath, $filePath);
        $this->cleanup($tempFilePath);

        Log::info('Proses periode selesai', ['file_path' => $filePath]);
    }

    /**
     * Validasi keberadaan file
     *
     * @param mixed $disk Storage disk instance
     * @param string $filePath Path file yang akan divalidasi
     * @return bool
     */
    private function validateFile($disk, string $filePath): bool
    {
        Log::info("Processing statement entry detail file: $filePath");

        if (!$disk->exists($filePath)) {
            Log::warning("File not found: $filePath");
            return false;
        }

        Log::info("File validated successfully: $filePath");
        return true;
    }

    /**
     * Buat file temporary untuk proses
     *
     * @param mixed $disk Storage disk instance
     * @param string $filePath Path file sumber
     * @param string $filename Nama file
     * @return string Path file temporary
     */
    private function createTemporaryFile($disk, string $filePath, string $filename): string
    {
        $tempFilePath = storage_path("app/temp_$filename");
        file_put_contents($tempFilePath, $disk->get($filePath));

        Log::info('Temporary file created', ['temp_path' => $tempFilePath]);
        return $tempFilePath;
    }

    /**
     * Proses file CSV
     *
     * @param string $tempFilePath Path file temporary
     * @param string $filePath Path file asli
     * @return void
     */
    private function processFile(string $tempFilePath, string $filePath): void
    {
        $handle = fopen($tempFilePath, "r");
        if ($handle === false) {
            Log::error("Unable to open file: $filePath");
            return;
        }

        $headers  = (new StmtEntryDetail())->getFillable();
        // Tambahkan field 'id' ke headers untuk menangani kolom tambahan di akhir CSV
        $expectedHeaders = array_merge($headers, ['id']);
        $rowCount = 0;
        $chunkCount = 0;

        Log::info('Memulai proses file', [
            'file_path' => $filePath,
            'headers_count' => count($headers),
            'expected_headers_count' => count($expectedHeaders)
        ]);

        while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) {
            $rowCount++;
            $this->processRow($row, $expectedHeaders, $rowCount, $filePath);

            // Process in chunks to avoid memory issues
            if (count($this->entryBatch) >= self::CHUNK_SIZE) {
                $this->saveBatch();
                $chunkCount++;
                Log::info("Processed chunk $chunkCount ({$this->processedCount} records so far)");
            }
        }

        // Process any remaining records
        if (!empty($this->entryBatch)) {
            $this->saveBatch();
        }

        fclose($handle);
        Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors.");
    }

    /**
     * Proses setiap baris data dengan penanganan field id tambahan
     *
     * @param array $row Data baris
     * @param array $expectedHeaders Header kolom yang diharapkan (termasuk id)
     * @param int $rowCount Nomor baris
     * @param string $filePath Path file
     * @return void
     */
    private function processRow(array $row, array $expectedHeaders, int $rowCount, string $filePath): void
    {
        // Validasi jumlah kolom - sekarang menggunakan expectedHeaders yang sudah include field 'id'
        if (count($expectedHeaders) !== count($row)) {
            Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " .
                count($expectedHeaders) . ", Got: " . count($row));
            $this->errorCount++;
            return;
        }

        // Kombinasikan data dengan headers
        $data = array_combine($expectedHeaders, $row);

        // Log untuk debugging struktur data
        Log::debug('Processing row data', [
            'row_count' => $rowCount,
            'stmt_entry_id' => $data['stmt_entry_id'] ?? 'not_set',
            'id' => $data['id'] ?? 'not_set'
        ]);

        // Logika untuk menggunakan field 'id' sebagai fallback jika stmt_entry_id kosong
        $this->handleStmtEntryIdFallback($data);

        // Hapus field 'id' dari data sebelum disimpan karena tidak ada di fillable model
        unset($data['id']);

        $this->cleanTransReference($data);
        $this->addToBatch($data, $rowCount, $filePath);
    }

    /**
     * Menangani logika fallback untuk stmt_entry_id menggunakan field id
     *
     * @param array $data Data yang akan diproses
     * @return void
     */
    private function handleStmtEntryIdFallback(array &$data): void
    {
        // Jika stmt_entry_id kosong atau null, gunakan value dari field 'id'
        if (empty($data['stmt_entry_id']) || $data['stmt_entry_id'] === '' || $data['stmt_entry_id'] === null) {
            if (isset($data['id']) && !empty($data['id'])) {
                $data['stmt_entry_id'] = $data['id'];

                Log::info('Using id as stmt_entry_id fallback', [
                    'original_stmt_entry_id' => $data['stmt_entry_id'] ?? 'empty',
                    'fallback_id' => $data['id']
                ]);
            } else {
                Log::warning('Both stmt_entry_id and id are empty', [
                    'stmt_entry_id' => $data['stmt_entry_id'] ?? 'not_set',
                    'id' => $data['id'] ?? 'not_set'
                ]);
            }
        }
    }

    /**
     * Tambahkan record ke batch untuk proses bulk insert
     *
     * @param array $data Data record
     * @param int $rowCount Nomor baris
     * @param string $filePath Path file
     * @return void
     */
    private function addToBatch(array $data, int $rowCount, string $filePath): void
    {
        try {
            // Validasi bahwa stmt_entry_id tidak kosong dan bukan header
            if (isset($data['stmt_entry_id']) &&
                $data['stmt_entry_id'] !== 'stmt_entry_id' &&
                !empty($data['stmt_entry_id'])) {

                // Add timestamp fields
                $now = now();
                $data['created_at'] = $now;
                $data['updated_at'] = $now;

                // Add to entry batch
                $this->entryBatch[] = $data;
                $this->processedCount++;

                Log::debug('Record added to batch', [
                    'row' => $rowCount,
                    'stmt_entry_id' => $data['stmt_entry_id']
                ]);
            } else {
                Log::warning('Skipping row due to invalid stmt_entry_id', [
                    'row' => $rowCount,
                    'stmt_entry_id' => $data['stmt_entry_id'] ?? 'not_set'
                ]);
                $this->errorCount++;
            }
        } catch (Exception $e) {
            $this->errorCount++;
            Log::error("Error processing Statement Entry Detail at row $rowCount in $filePath: " . $e->getMessage());
        }
    }

    /**
     * Bersihkan trans_reference dari karakter tidak diinginkan
     *
     * @param array $data Data yang akan dibersihkan
     * @return void
     */
    private function cleanTransReference(array &$data): void
    {
        if (isset($data['trans_reference'])) {
            // Clean trans_reference from \\BNK if present
            $data['trans_reference'] = preg_replace('/\\\\.*$/', '', $data['trans_reference']);
            Log::debug('Trans reference cleaned', ['original' => $data['trans_reference']]);
        }
    }

    /**
     * Simpan batch data ke database menggunakan updateOrCreate
     * untuk menghindari error unique constraint
     *
     * @return void
     * @throws Exception
     */
    private function saveBatch(): void
    {
        Log::info('Memulai proses saveBatch dengan updateOrCreate');

        DB::beginTransaction();

        try {
            if (!empty($this->entryBatch)) {
                $totalProcessed = 0;

                // Process each entry data directly
                foreach ($this->entryBatch as $entryData) {
                    // Validasi bahwa entryData adalah array dan memiliki stmt_entry_id
                    if (is_array($entryData) && isset($entryData['stmt_entry_id'])) {
                        // Gunakan updateOrCreate untuk menghindari duplicate key error
                        StmtEntryDetail::updateOrCreate(
                            [
                                'stmt_entry_id' => $entryData['stmt_entry_id']
                            ],
                            $entryData
                        );

                        $totalProcessed++;
                    } else {
                        Log::warning('Invalid entry data structure', ['data' => $entryData]);
                        $this->errorCount++;
                    }
                }

                DB::commit();

                Log::info("Berhasil memproses {$totalProcessed} record dengan updateOrCreate");

                // Reset entry batch after successful processing
                $this->entryBatch = [];
            }
        } catch (Exception $e) {
            DB::rollback();

            Log::error("Error in saveBatch: " . $e->getMessage() . "\n" . $e->getTraceAsString());
            $this->errorCount += count($this->entryBatch);

            // Reset batch even if there's an error to prevent reprocessing the same failed records
            $this->entryBatch = [];

            throw $e;
        }
    }

    /**
     * Bersihkan file temporary
     *
     * @param string $tempFilePath Path file temporary
     * @return void
     */
    private function cleanup(string $tempFilePath): void
    {
        if (file_exists($tempFilePath)) {
            unlink($tempFilePath);
            Log::info('Temporary file cleaned up', ['temp_path' => $tempFilePath]);
        }
    }

    /**
     * Log penyelesaian job
     *
     * @return void
     */
    private function logJobCompletion(): void
    {
        Log::info("Statement Entry Detail data processing completed. " .
            "Total processed: {$this->processedCount}, Total errors: {$this->errorCount}");
    }
}