webstatement/app/Jobs/ProcessDataCaptureDataJob.php

<?php

    namespace Modules\Webstatement\Jobs;

    use Exception;
    use Illuminate\Bus\Queueable;
    use Illuminate\Contracts\Queue\ShouldQueue;
    use Illuminate\Foundation\Bus\Dispatchable;
    use Illuminate\Queue\InteractsWithQueue;
    use Illuminate\Queue\SerializesModels;
    use Illuminate\Support\Facades\Log;
    use Illuminate\Support\Facades\Storage;
    use Modules\Webstatement\Models\DataCapture;

    class ProcessDataCaptureDataJob implements ShouldQueue
    {
        use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;

        protected $periods;
        protected $filename;

        /**
         * Create a new job instance.
         */
        public function __construct(array $periods = [], string $filename = "ST.DATA.CAPTURE.csv")
        {
            $this->periods  = $periods;
            $this->filename = $filename;
        }

        /**
         * Execute the job.
         */
        public function handle()
        : void
        {
            try {
                set_time_limit(24 * 60 * 60);
                $disk           = Storage::disk('sftpStatement');
                $processedCount = 0;
                $errorCount     = 0;

                if (empty($this->periods)) {
                    Log::warning('No periods provided for data capture processing');
                    return;
                }

                foreach ($this->periods as $period) {
                    // Skip the _parameter folder
                    if ($period === '_parameter') {
                        Log::info("Skipping _parameter folder");
                        continue;
                    }

                    // Construct the filepath based on the period folder name
                    $fileName = "$period.$this->filename";
                    $filePath = "$period/$fileName";

                    Log::info("Processing data capture file: $filePath");

                    if (!$disk->exists($filePath)) {
                        Log::warning("File not found: $filePath");
                        continue;
                    }

                    // Create a temporary local copy of the file
                    $tempFilePath = storage_path("app/temp_{$this->filename}");
                    file_put_contents($tempFilePath, $disk->get($filePath));

                    $handle = fopen($tempFilePath, "r");

                    if ($handle !== false) {
                        // CSV headers from the file
                        $csvHeaders = [
                            'id',
                            'account_number',
                            'sign',
                            'amount_lcy',
                            'transaction_code',
                            'their_reference',
                            'narrative',
                            'pl_category',
                            'customer_id',
                            'account_officer',
                            'product_category',
                            'value_date',
                            'currency',
                            'amount_fcy',
                            'exchange_rate',
                            'neg_ref_no',
                            'position_type',
                            'our_reference',
                            'reversal_marker',
                            'exposure_date',
                            'currency_market',
                            'iblc_country',
                            'last_version',
                            'otor_version',
                            'department_code',
                            'dealer_desk',
                            'bank_sort_cde',
                            'cheque_number',
                            'accounting_date',
                            'contingent_acct',
                            'cheq_type',
                            'tfs_reference',
                            'accounting_company',
                            'stmt_no',
                            'curr_no',
                            'inputter',
                            'authoriser',
                            'co_code',
                            'date_time'
                        ];

                        $rowCount = 0;

                        while (($row = fgetcsv($handle, 0, "~")) !== false) {
                            $rowCount++;

                            // Skip header row if it exists
                            if ($rowCount === 1 && strtolower($row[0]) === 'id') {
                                continue;
                            }

                            if (count($csvHeaders) === count($row)) {
                                $data = array_combine($csvHeaders, $row);

                                try {
                                    // Format dates if they exist
                                    foreach (['value_date', 'exposure_date', 'accounting_date'] as $dateField) {
                                        if (!empty($data[$dateField])) {
                                            try {
                                                $data[$dateField] = date('Y-m-d', strtotime($data[$dateField]));
                                            } catch (Exception $e) {
                                                // If date parsing fails, keep the original value
                                                Log::warning("Failed to parse date for $dateField: {$data[$dateField]}");
                                            }
                                        }
                                    }

                                    // Format datetime if it exists
                                    if (!empty($data['date_time'])) {
                                        try {
                                            $data['date_time'] = date('Y-m-d H:i:s', strtotime($data['date_time']));
                                        } catch (Exception $e) {
                                            // If datetime parsing fails, keep the original value
                                            Log::warning("Failed to parse datetime for date_time: {$data['date_time']}");
                                        }
                                    }

                                    if (!empty($data['id'])) {
                                        DataCapture::updateOrCreate(
                                            ['id' => $data['id']],
                                            $data
                                        );
                                        $processedCount++;
                                    }
                                } catch (Exception $e) {
                                    $errorCount++;
                                    Log::error("Error processing Data Capture at row $rowCount in $filePath: " . $e->getMessage());
                                }
                            } else {
                                Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($csvHeaders) . ", Got: " . count($row));
                            }
                        }

                        fclose($handle);
                        Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors.");

                        // Clean up the temporary file
                        unlink($tempFilePath);
                    } else {
                        Log::error("Unable to open file: $filePath");
                    }
                }

                Log::info("Data capture processing completed. Total processed: $processedCount, Total errors: $errorCount");

            } catch (Exception $e) {
                Log::error('Error in ProcessDataCaptureDataJob: ' . $e->getMessage());
                throw $e;
            }
        }
    }