period = $period; $this->filename = $filename; } /** * Execute the job. */ public function handle() : void { try { set_time_limit(24 * 60 * 60); $disk = Storage::disk('sftpStatement'); $processedCount = 0; $errorCount = 0; if ($this->period === '') { Log::warning('No period provided for data capture processing'); return; } // Construct the filepath based on the period folder name $fileName = "{$this->period}.{$this->filename}"; $filePath = "{$this->period}/$fileName"; Log::info("Processing data capture file: $filePath"); if (!$disk->exists($filePath)) { Log::warning("File not found: $filePath"); return; } // Create a temporary local copy of the file $tempFilePath = storage_path("app/temp_{$this->filename}"); file_put_contents($tempFilePath, $disk->get($filePath)); $handle = fopen($tempFilePath, "r"); if ($handle !== false) { // CSV headers from the file $csvHeaders = [ 'id', 'account_number', 'sign', 'amount_lcy', 'transaction_code', 'their_reference', 'narrative', 'pl_category', 'customer_id', 'account_officer', 'product_category', 'value_date', 'currency', 'amount_fcy', 'exchange_rate', 'neg_ref_no', 'position_type', 'our_reference', 'reversal_marker', 'exposure_date', 'currency_market', 'iblc_country', 'last_version', 'otor_version', 'department_code', 'dealer_desk', 'bank_sort_cde', 'cheque_number', 'accounting_date', 'contingent_acct', 'cheq_type', 'tfs_reference', 'accounting_company', 'stmt_no', 'curr_no', 'inputter', 'authoriser', 'co_code', 'date_time' ]; $rowCount = 0; while (($row = fgetcsv($handle, 0, "~")) !== false) { $rowCount++; // Skip header row if it exists if ($rowCount === 1 && strtolower($row[0]) === 'id') { continue; } if (count($csvHeaders) === count($row)) { $data = array_combine($csvHeaders, $row); try { // Format dates if they exist foreach (['value_date', 'exposure_date', 'accounting_date'] as $dateField) { if (!empty($data[$dateField])) { try { $data[$dateField] = date('Y-m-d', strtotime($data[$dateField])); } catch (Exception $e) { // If date parsing fails, keep the original value Log::warning("Failed to parse date for $dateField: {$data[$dateField]}"); } } } // Format datetime if it exists if (!empty($data['date_time'])) { try { $data['date_time'] = date('Y-m-d H:i:s', strtotime($data['date_time'])); } catch (Exception $e) { // If datetime parsing fails, keep the original value Log::warning("Failed to parse datetime for date_time: {$data['date_time']}"); } } if (!empty($data['id'])) { DataCapture::updateOrCreate( ['id' => $data['id']], $data ); $processedCount++; } } catch (Exception $e) { $errorCount++; Log::error("Error processing Data Capture at row $rowCount in $filePath: " . $e->getMessage()); } } else { Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($csvHeaders) . ", Got: " . count($row)); } } fclose($handle); Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors."); // Clean up the temporary file unlink($tempFilePath); } else { Log::error("Unable to open file: $filePath"); } Log::info("Data capture processing completed. Total processed: $processedCount, Total errors: $errorCount"); } catch (Exception $e) { Log::error('Error in ProcessDataCaptureDataJob: ' . $e->getMessage()); throw $e; } } }