webstatement/app/Jobs/ProcessCustomerDataJob.php

<?php

    namespace Modules\Webstatement\Jobs;

    use Exception;
    use Illuminate\Bus\Queueable;
    use Illuminate\Contracts\Queue\ShouldQueue;
    use Illuminate\Foundation\Bus\Dispatchable;
    use Illuminate\Queue\InteractsWithQueue;
    use Illuminate\Queue\SerializesModels;
    use Illuminate\Support\Facades\DB;
    use Illuminate\Support\Facades\Log;
    use Illuminate\Support\Facades\Storage;
    use Modules\Webstatement\Models\Customer;

    class ProcessCustomerDataJob implements ShouldQueue
    {
        use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;

        private const CSV_DELIMITER      = '~';
        private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds
        private const FILENAME           = 'ST.CUSTOMER.csv';
        private const DISK_NAME          = 'staging';
        private const CHUNK_SIZE         = 1000; // Process data in chunks to reduce memory usage

        private string $period = '';
        private int    $processedCount = 0;
        private int    $errorCount     = 0;
        private array  $customerBatch  = [];

        /**
         * Create a new job instance.
         */
        public function __construct(string $period = '')
        {
            $this->period = $period;
        }

        /**
         * Execute the job.
         */
        public function handle()
        : void
        {
            try {
                $this->initializeJob();

                if ($this->period === '') {
                    Log::warning('No period provided for customer data processing');
                    return;
                }

                $this->processPeriod();
                $this->logJobCompletion();
            } catch (Exception $e) {
                Log::error('Error in ProcessCustomerDataJob: ' . $e->getMessage());
                throw $e;
            }
        }

        private function initializeJob()
        : void
        {
            set_time_limit(self::MAX_EXECUTION_TIME);
            $this->processedCount = 0;
            $this->errorCount     = 0;
            $this->customerBatch  = [];
        }

        private function processPeriod()
        : void
        {
            $disk     = Storage::disk(self::DISK_NAME);
            $filename = "{$this->period}." . self::FILENAME;
            $filePath = "{$this->period}/$filename";

            if (!$this->validateFile($disk, $filePath)) {
                return;
            }

            $tempFilePath = $this->createTemporaryFile($disk, $filePath, $filename);
            $this->processFile($tempFilePath, $filePath);
            $this->cleanup($tempFilePath);
        }

        private function validateFile($disk, string $filePath)
        : bool
        {
            Log::info("Processing customer file: $filePath");

            if (!$disk->exists($filePath)) {
                Log::warning("File not found: $filePath");
                return false;
            }

            return true;
        }

        private function createTemporaryFile($disk, string $filePath, string $filename)
        : string
        {
            $tempFilePath = storage_path("app/temp_$filename");
            file_put_contents($tempFilePath, $disk->get($filePath));
            return $tempFilePath;
        }

        private function processFile(string $tempFilePath, string $filePath)
        : void
        {
            $handle = fopen($tempFilePath, "r");
            if ($handle === false) {
                Log::error("Unable to open file: $filePath");
                return;
            }

            // Read header from CSV file
            $csvHeaders = fgetcsv($handle, 0, self::CSV_DELIMITER);
            if ($csvHeaders === false) {
                Log::error("Unable to read headers from file: $filePath");
                fclose($handle);
                return;
            }

            // Map CSV headers to database fields
            $headerMapping = $this->getHeaderMapping($csvHeaders);
            
            Log::info("CSV Headers found", [
                'csv_headers' => $csvHeaders,
                'mapped_fields' => array_values($headerMapping)
            ]);

            $rowCount = 0;
            $chunkCount = 0;

            while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) {
                $rowCount++;
                $this->processRow($row, $csvHeaders, $headerMapping, $rowCount, $filePath);

                // Process in chunks to avoid memory issues
                if (count($this->customerBatch) >= self::CHUNK_SIZE) {
                    $this->saveBatch();
                    $chunkCount++;
                    Log::info("Processed chunk $chunkCount ({$this->processedCount} records so far)");
                }
            }

            // Process any remaining records
            if (!empty($this->customerBatch)) {
                $this->saveBatch();
            }

            fclose($handle);
            Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors.");
        }

        /**
         * Map CSV headers to database field names
         * Memetakan header CSV ke nama field database
         */
        private function getHeaderMapping(array $csvHeaders): array
        {
            $mapping = [];
            $fillableFields = (new Customer())->getFillable();
            
            foreach ($csvHeaders as $index => $csvHeader) {
                $csvHeader = trim($csvHeader);
                
                // Direct mapping untuk field yang sama
                if (in_array($csvHeader, $fillableFields)) {
                    $mapping[$index] = $csvHeader;
                    continue;
                }

                // Custom mapping untuk field yang berbeda nama
                $customMapping = [
                    'co_code' => 'branch_code', // co_code di CSV menjadi branch_code di database
                    'name_1' => 'name'
                ];

                if (isset($customMapping[$csvHeader])) {
                    $mapping[$index] = $customMapping[$csvHeader];
                } else {
                    // Jika field ada di fillable, gunakan langsung
                    if (in_array($csvHeader, $fillableFields)) {
                        $mapping[$index] = $csvHeader;
                    }
                    // Jika tidak ada mapping, skip field ini
                }
            }
            
            return $mapping;
        }

        private function processRow(array $row, array $csvHeaders, array $headerMapping, int $rowCount, string $filePath)
        : void
        {
            if (count($csvHeaders) !== count($row)) {
                Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " .
                    count($csvHeaders) . ", Got: " . count($row));
                return;
            }

            // Map CSV data to database fields
            $data = [];
            foreach ($row as $index => $value) {
                if (isset($headerMapping[$index])) {
                    $fieldName = $headerMapping[$index];
                    $data[$fieldName] = trim($value);
                }
            }
            
            $this->addToBatch($data, $rowCount, $filePath);
        }

        /**
         * Add record to batch instead of saving immediately
         */
        private function addToBatch(array $data, int $rowCount, string $filePath)
        : void
        {
            try {
                if (isset($data['customer_code']) && $data['customer_code'] !== 'customer_code') {
                    // Add timestamp fields
                    $now = now();
                    $data['created_at'] = $now;
                    $data['updated_at'] = $now;

                    // Add to customer batch
                    $this->customerBatch[] = $data;
                    $this->processedCount++;
                }
            } catch (Exception $e) {
                $this->errorCount++;
                Log::error("Error processing Customer at row $rowCount in $filePath: " . $e->getMessage());
            }
        }

        /**
         * Save batched records to the database
         * Menyimpan data customer dalam batch ke database dengan transaksi
         */
        private function saveBatch()
        : void
        {
            if (empty($this->customerBatch)) {
                return;
            }

            $batchSize = count($this->customerBatch);
            Log::info("Starting batch save", ['batch_size' => $batchSize]);

            try {
                DB::transaction(function () use ($batchSize) {
                    // Bulk insert/update customers
                    Customer::upsert(
                        $this->customerBatch,
                        ['customer_code'], // Unique key
                        array_diff((new Customer())->getFillable(), ['customer_code']) // Update columns
                    );

                    Log::info("Batch save completed successfully", ['batch_size' => $batchSize]);
                });

                // Reset customer batch after successful processing
                $this->customerBatch = [];
                
            } catch (Exception $e) {
                Log::error("Error in saveBatch", [
                    'error' => $e->getMessage(),
                    'batch_size' => $batchSize,
                    'trace' => $e->getTraceAsString()
                ]);
                
                $this->errorCount += $batchSize;
                
                // Reset batch even if there's an error to prevent reprocessing the same failed records
                $this->customerBatch = [];
                
                // Re-throw exception untuk handling di level atas
                throw $e;
            }
        }

        private function cleanup(string $tempFilePath)
        : void
        {
            if (file_exists($tempFilePath)) {
                unlink($tempFilePath);
            }
        }

        private function logJobCompletion()
        : void
        {
            Log::info("Customer data processing completed. " .
                "Total processed: {$this->processedCount}, Total errors: {$this->errorCount}");
        }
    }