diff --git a/app/Jobs/ProcessAccountDataJob.php b/app/Jobs/ProcessAccountDataJob.php index 82a214d..067df63 100644 --- a/app/Jobs/ProcessAccountDataJob.php +++ b/app/Jobs/ProcessAccountDataJob.php @@ -17,7 +17,13 @@ { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; - protected $period; + private const CSV_DELIMITER = '~'; + private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds + private const FILENAME = 'ST.ACCOUNT.csv'; + private const DISK_NAME = 'sftpStatement'; + private string $period; + private int $processedCount = 0; + private int $errorCount = 0; /** * Create a new job instance. @@ -34,107 +40,172 @@ : void { try { - set_time_limit(24 * 60 * 60); - $disk = Storage::disk('sftpStatement'); - $processedCount = 0; - $errorCount = 0; + $this->initializeJob(); if ($this->period === '') { Log::warning('No period provided for account data processing'); return; } - // Construct the filename based on the period folder name - $filename = "{$this->period}.ST.ACCOUNT.csv"; - $filePath = "{$this->period}/$filename"; - - Log::info("Processing account file: $filePath"); - - if (!$disk->exists($filePath)) { - Log::warning("File not found: $filePath"); - return; - } - - // Create a temporary local copy of the file - $tempFilePath = storage_path("app/temp_$filename"); - file_put_contents($tempFilePath, $disk->get($filePath)); - - $handle = fopen($tempFilePath, "r"); - - if ($handle !== false) { - $headers = (new Account())->getFillable(); - Log::info('Headers: ' . implode(", ", $headers)); - $rowCount = 0; - - while (($row = fgetcsv($handle, 0, "~")) !== false) { - $rowCount++; - - if (count($headers) === count($row)) { - $data = array_combine($headers, $row); - - // Check if start_year_bal is empty and set it to 0 if so - if (empty($data['start_year_bal']) || $data['start_year_bal'] == "" || $data['start_year_bal'] == null) { - $data['start_year_bal'] = 0; - } - - if (empty($data['closure_date']) || $data['closure_date'] == "" || $data['closure_date'] == null) { - $data['closure_date'] = null; - } - - try { - if ($data['account_number'] !== 'account_number') { - // Use firstOrNew instead of updateOrCreate - $account = Account::firstOrNew(['account_number' => $data['account_number']]); - $account->fill($data); - $account->save(); - - // Store the opening balances in the AccountBalance model for this period - if (isset($data['open_actual_bal']) || isset($data['open_cleared_bal'])) { - // Prepare balance data for bulk insert/update - $balanceData = [ - 'account_number' => $data['account_number'], - 'period' => $this->period, - 'actual_balance' => empty($data['open_actual_bal']) ? 0 : $data['open_actual_bal'], - 'cleared_balance' => empty($data['open_cleared_bal']) ? 0 : $data['open_cleared_bal'], - 'created_at' => now(), - 'updated_at' => now() - ]; - - // Use updateOrInsert to reduce queries - AccountBalance::updateOrInsert( - [ - 'account_number' => $data['account_number'], - 'period' => $this->period - ], - $balanceData - ); - } - - $processedCount++; - } - } catch (Exception $e) { - $errorCount++; - Log::error("Error processing Account at row $rowCount in $filePath: " . $e->getMessage()); - } - } else { - Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row)); - } - } - - fclose($handle); - Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors."); - - // Clean up the temporary file - unlink($tempFilePath); - } else { - Log::error("Unable to open file: $filePath"); - } - - Log::info("Account data processing completed. Total processed: $processedCount, Total errors: $errorCount"); - + $this->processPeriod(); + $this->logJobCompletion(); } catch (Exception $e) { Log::error('Error in ProcessAccountDataJob: ' . $e->getMessage()); throw $e; } } + + private function initializeJob() + : void + { + set_time_limit(self::MAX_EXECUTION_TIME); + $this->processedCount = 0; + $this->errorCount = 0; + } + + private function processPeriod() + : void + { + $disk = Storage::disk(self::DISK_NAME); + $filename = "{$this->period}." . self::FILENAME; + $filePath = "{$this->period}/$filename"; + + if (!$this->validateFile($disk, $filePath)) { + return; + } + + $tempFilePath = $this->createTemporaryFile($disk, $filePath, $filename); + $this->processFile($tempFilePath, $filePath); + $this->cleanup($tempFilePath); + } + + private function validateFile($disk, string $filePath) + : bool + { + Log::info("Processing account file: $filePath"); + + if (!$disk->exists($filePath)) { + Log::warning("File not found: $filePath"); + return false; + } + + return true; + } + + private function createTemporaryFile($disk, string $filePath, string $filename) + : string + { + $tempFilePath = storage_path("app/temp_$filename"); + file_put_contents($tempFilePath, $disk->get($filePath)); + return $tempFilePath; + } + + private function processFile(string $tempFilePath, string $filePath) + : void + { + $handle = fopen($tempFilePath, "r"); + if ($handle === false) { + Log::error("Unable to open file: $filePath"); + return; + } + + $headers = (new Account())->getFillable(); + Log::info('Headers: ' . implode(", ", $headers)); + $rowCount = 0; + + while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) { + $rowCount++; + $this->processRow($row, $headers, $rowCount, $filePath); + } + + fclose($handle); + Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors."); + } + + private function processRow(array $row, array $headers, int $rowCount, string $filePath) + : void + { + if (count($headers) !== count($row)) { + Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . + count($headers) . ", Got: " . count($row)); + return; + } + + $data = array_combine($headers, $row); + $this->normalizeData($data); + $this->saveRecord($data, $rowCount, $filePath); + } + + private function normalizeData(array &$data) + : void + { + // Check if start_year_bal is empty and set it to 0 if so + if (empty($data['start_year_bal']) || $data['start_year_bal'] == "" || $data['start_year_bal'] == null) { + $data['start_year_bal'] = 0; + } + + if (empty($data['closure_date']) || $data['closure_date'] == "" || $data['closure_date'] == null) { + $data['closure_date'] = null; + } + } + + private function saveRecord(array $data, int $rowCount, string $filePath) + : void + { + try { + if ($data['account_number'] !== 'account_number') { + // Use firstOrNew instead of updateOrCreate + $account = Account::firstOrNew(['account_number' => $data['account_number']]); + $account->fill($data); + $account->save(); + + $this->saveAccountBalance($data); + $this->processedCount++; + } + } catch (Exception $e) { + $this->errorCount++; + Log::error("Error processing Account at row $rowCount in $filePath: " . $e->getMessage()); + } + } + + private function saveAccountBalance(array $data) + : void + { + // Store the opening balances in the AccountBalance model for this period + if (isset($data['open_actual_bal']) || isset($data['open_cleared_bal'])) { + // Prepare balance data for bulk insert/update + $balanceData = [ + 'account_number' => $data['account_number'], + 'period' => $this->period, + 'actual_balance' => empty($data['open_actual_bal']) ? 0 : $data['open_actual_bal'], + 'cleared_balance' => empty($data['open_cleared_bal']) ? 0 : $data['open_cleared_bal'], + 'created_at' => now(), + 'updated_at' => now() + ]; + + // Use updateOrInsert to reduce queries + AccountBalance::updateOrInsert( + [ + 'account_number' => $data['account_number'], + 'period' => $this->period + ], + $balanceData + ); + } + } + + private function cleanup(string $tempFilePath) + : void + { + if (file_exists($tempFilePath)) { + unlink($tempFilePath); + } + } + + private function logJobCompletion() + : void + { + Log::info("Account data processing completed. " . + "Total processed: {$this->processedCount}, Total errors: {$this->errorCount}"); + } } diff --git a/app/Jobs/ProcessArrangementDataJob.php b/app/Jobs/ProcessArrangementDataJob.php index 238bb6f..2c30596 100644 --- a/app/Jobs/ProcessArrangementDataJob.php +++ b/app/Jobs/ProcessArrangementDataJob.php @@ -16,7 +16,14 @@ { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; - protected string $period; + private const CSV_DELIMITER = '~'; + private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds + private const FILENAME = 'ST.AA.ARRANGEMENT.csv'; + private const DISK_NAME = 'sftpStatement'; + + private string $period; + private int $processedCount = 0; + private int $errorCount = 0; /** * Create a new job instance. @@ -29,76 +36,133 @@ /** * Execute the job. */ - public function handle(): void + public function handle() + : void { try { - set_time_limit(24 * 60 * 60); - $disk = Storage::disk('sftpStatement'); - $processedCount = 0; - $errorCount = 0; + $this->initializeJob(); if ($this->period === '') { Log::warning('No period provided for arrangement data processing'); return; } - // Construct the filename based on the period folder name - $filename = "{$this->period}.ST.AA.ARRANGEMENT.csv"; - $filePath = "{$this->period}/$filename"; - - Log::info("Processing arrangement file: $filePath"); - - if (!$disk->exists($filePath)) { - Log::warning("File not found: $filePath"); - return; - } - - // Create a temporary local copy of the file - $tempFilePath = storage_path("app/temp_$filename"); - file_put_contents($tempFilePath, $disk->get($filePath)); - - $handle = fopen($tempFilePath, "r"); - - if ($handle !== false) { - $headers = (new TempArrangement())->getFillable(); - $rowCount = 0; - - while (($row = fgetcsv($handle, 0, "~")) !== false) { - $rowCount++; - - if (count($headers) === count($row)) { - $data = array_combine($headers, $row); - try { - if ($data['arrangement_id'] !== 'arrangement_id') { - TempArrangement::updateOrCreate( - ['arrangement_id' => $data['arrangement_id']], // key to find existing record - $data // data to update or create - ); - $processedCount++; - } - } catch (Exception $e) { - $errorCount++; - Log::error("Error processing Arrangement at row $rowCount in $filePath: " . $e->getMessage()); - } - } else { - Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row)); - } - } - - fclose($handle); - Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors."); - - // Clean up the temporary file - unlink($tempFilePath); - } else { - Log::error("Unable to open file: $filePath"); - } - - Log::info("Arrangement data processing completed. Total processed: $processedCount, Total errors: $errorCount"); - + $this->processPeriod(); + $this->logJobCompletion(); } catch (Exception $e) { Log::error('Error in ProcessArrangementDataJob: ' . $e->getMessage()); throw $e; } } + + private function initializeJob() + : void + { + set_time_limit(self::MAX_EXECUTION_TIME); + $this->processedCount = 0; + $this->errorCount = 0; + } + + private function processPeriod() + : void + { + $disk = Storage::disk(self::DISK_NAME); + $filename = "{$this->period}." . self::FILENAME; + $filePath = "{$this->period}/$filename"; + + if (!$this->validateFile($disk, $filePath)) { + return; + } + + $tempFilePath = $this->createTemporaryFile($disk, $filePath, $filename); + $this->processFile($tempFilePath, $filePath); + $this->cleanup($tempFilePath); + } + + private function validateFile($disk, string $filePath) + : bool + { + Log::info("Processing arrangement file: $filePath"); + + if (!$disk->exists($filePath)) { + Log::warning("File not found: $filePath"); + return false; + } + + return true; + } + + private function createTemporaryFile($disk, string $filePath, string $filename) + : string + { + $tempFilePath = storage_path("app/temp_$filename"); + file_put_contents($tempFilePath, $disk->get($filePath)); + return $tempFilePath; + } + + private function processFile(string $tempFilePath, string $filePath) + : void + { + $handle = fopen($tempFilePath, "r"); + if ($handle === false) { + Log::error("Unable to open file: $filePath"); + return; + } + + $headers = (new TempArrangement())->getFillable(); + $rowCount = 0; + + while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) { + $rowCount++; + $this->processRow($row, $headers, $rowCount, $filePath); + } + + fclose($handle); + Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors."); + } + + private function processRow(array $row, array $headers, int $rowCount, string $filePath) + : void + { + if (count($headers) !== count($row)) { + Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . + count($headers) . ", Got: " . count($row)); + return; + } + + $data = array_combine($headers, $row); + $this->saveRecord($data, $rowCount, $filePath); + } + + private function saveRecord(array $data, int $rowCount, string $filePath) + : void + { + try { + if ($data['arrangement_id'] !== 'arrangement_id') { + TempArrangement::updateOrCreate( + ['arrangement_id' => $data['arrangement_id']], // key to find existing record + $data // data to update or create + ); + $this->processedCount++; + } + } catch (Exception $e) { + $this->errorCount++; + Log::error("Error processing Arrangement at row $rowCount in $filePath: " . $e->getMessage()); + } + } + + private function cleanup(string $tempFilePath) + : void + { + if (file_exists($tempFilePath)) { + unlink($tempFilePath); + } + } + + private function logJobCompletion() + : void + { + Log::info("Arrangement data processing completed. " . + "Total processed: {$this->processedCount}, Total errors: {$this->errorCount}"); + } } diff --git a/app/Jobs/ProcessAtmTransactionJob.php b/app/Jobs/ProcessAtmTransactionJob.php index be6e2b8..c51cde6 100644 --- a/app/Jobs/ProcessAtmTransactionJob.php +++ b/app/Jobs/ProcessAtmTransactionJob.php @@ -16,11 +16,11 @@ { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; - // Konstanta untuk nilai-nilai statis - private const FILE_EXTENSION = '.ST.ATM.TRANSACTION.csv'; - private const CSV_DELIMITER = '~'; - private const DISK_NAME = 'sftpStatement'; - private const HEADER_MAP = [ + private const CSV_DELIMITER = '~'; + private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds + private const FILENAME = 'ST.ATM.TRANSACTION.csv'; + private const DISK_NAME = 'sftpStatement'; + private const HEADER_MAP = [ 'id' => 'transaction_id', 'card_acc_id' => 'card_acc_id', 'pan_number' => 'pan_number', @@ -40,8 +40,9 @@ 'proc_code' => 'proc_code' ]; - // Pemetaan bidang header ke kolom model - protected string $period; + private string $period; + private int $processedCount = 0; + private int $errorCount = 0; /** * Create a new job instance. @@ -58,64 +59,59 @@ : void { try { - set_time_limit(24 * 60 * 60); + $this->initializeJob(); if ($this->period === '') { Log::warning('No period provided for ATM transaction data processing'); return; } - $stats = $this->processPeriodFile(); - - Log::info("ProcessAtmTransactionJob completed. Total processed: {$stats['processed']}, Total errors: {$stats['errors']}"); + $this->processPeriod(); + $this->logJobCompletion(); } catch (Exception $e) { - Log::error("Error in ProcessAtmTransactionJob: " . $e->getMessage()); + Log::error('Error in ProcessAtmTransactionJob: ' . $e->getMessage()); throw $e; } } - /** - * Process a single period file - */ - private function processPeriodFile() - : array + private function initializeJob() + : void { - $disk = Storage::disk(self::DISK_NAME); - $filename = $this->period . self::FILE_EXTENSION; - $filePath = "{$this->period}/$filename"; - $processedCount = 0; - $errorCount = 0; + set_time_limit(self::MAX_EXECUTION_TIME); + $this->processedCount = 0; + $this->errorCount = 0; + } + private function processPeriod() + : void + { + $disk = Storage::disk(self::DISK_NAME); + $filename = "{$this->period}." . self::FILENAME; + $filePath = "{$this->period}/$filename"; + + if (!$this->validateFile($disk, $filePath)) { + return; + } + + $tempFilePath = $this->createTemporaryFile($disk, $filePath, $filename); + $this->processFile($tempFilePath, $filePath); + $this->cleanup($tempFilePath); + } + + private function validateFile($disk, string $filePath) + : bool + { Log::info("Processing ATM transaction file: $filePath"); if (!$disk->exists($filePath)) { Log::warning("File not found: $filePath"); - return ['processed' => 0, 'errors' => 0]; + return false; } - $tempFilePath = $this->createTempFile($disk, $filePath, $filename); - - $result = $this->processCSVFile($tempFilePath, $filePath); - $processedCount += $result['processed']; - $errorCount += $result['errors']; - - // Clean up the temporary file - if (file_exists($tempFilePath)) { - unlink($tempFilePath); - } - - Log::info("Completed processing $filePath. Processed {$result['processed']} records with {$result['errors']} errors."); - - return [ - 'processed' => $processedCount, - 'errors' => $errorCount - ]; + return true; } - /** - * Create a temporary file for processing - */ - private function createTempFile($disk, string $filePath, string $filename) + private function createTemporaryFile($disk, string $filePath, string $filename) : string { $tempFilePath = storage_path("app/temp_$filename"); @@ -123,59 +119,49 @@ return $tempFilePath; } - /** - * Process a CSV file and import data - */ - private function processCSVFile(string $tempFilePath, string $originalFilePath) - : array + private function processFile(string $tempFilePath, string $filePath) + : void { - $processedCount = 0; - $errorCount = 0; - $handle = fopen($tempFilePath, "r"); if ($handle === false) { - Log::error("Unable to open file: $originalFilePath"); - return ['processed' => 0, 'errors' => 0]; + Log::error("Unable to open file: $filePath"); + return; } // Get the headers from the first row $headerRow = fgetcsv($handle, 0, self::CSV_DELIMITER); if (!$headerRow) { fclose($handle); - return ['processed' => 0, 'errors' => 0]; + return; } $rowCount = 0; while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) { $rowCount++; - - if (count($headerRow) !== count($row)) { - Log::warning("Row $rowCount in $originalFilePath has incorrect column count. Expected: " . count($headerRow) . ", Got: " . count($row)); - continue; - } - - $result = $this->processRow($headerRow, $row, $rowCount, $originalFilePath); - $processedCount += $result['processed']; - $errorCount += $result['errors']; + $this->processRow($headerRow, $row, $rowCount, $filePath); } fclose($handle); - - return [ - 'processed' => $processedCount, - 'errors' => $errorCount - ]; + Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors."); } - /** - * Process a single row from the CSV file - */ private function processRow(array $headerRow, array $row, int $rowCount, string $filePath) - : array + : void { + if (count($headerRow) !== count($row)) { + Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . + count($headerRow) . ", Got: " . count($row)); + return; + } + // Combine the header row with the data row $rawData = array_combine($headerRow, $row); + $this->mapAndSaveRecord($rawData, $rowCount, $filePath); + } + private function mapAndSaveRecord(array $rawData, int $rowCount, string $filePath) + : void + { // Map the raw data to our model fields $data = []; foreach (self::HEADER_MAP as $csvField => $modelField) { @@ -184,18 +170,16 @@ // Skip header row if it was included in the data if ($data['transaction_id'] === 'id') { - return ['processed' => 0, 'errors' => 0]; + return; } - try { - // Format dates if needed - /*if (!empty($data['booking_date'])) { - $data['booking_date'] = date('Y-m-d H:i:s', strtotime($data['booking_date'])); - } + $this->saveRecord($data, $rowCount, $filePath); + } - if (!empty($data['value_date'])) { - $data['value_date'] = date('Y-m-d H:i:s', strtotime($data['value_date'])); - }*/ + private function saveRecord(array $data, int $rowCount, string $filePath) + : void + { + try { // Create or update the record AtmTransaction::updateOrCreate( @@ -203,10 +187,25 @@ $data ); - return ['processed' => 1, 'errors' => 0]; + $this->processedCount++; } catch (Exception $e) { + $this->errorCount++; Log::error("Error processing row $rowCount in $filePath: " . $e->getMessage()); - return ['processed' => 0, 'errors' => 1]; } } + + private function cleanup(string $tempFilePath) + : void + { + if (file_exists($tempFilePath)) { + unlink($tempFilePath); + } + } + + private function logJobCompletion() + : void + { + Log::info("ATM transaction data processing completed. " . + "Total processed: {$this->processedCount}, Total errors: {$this->errorCount}"); + } } diff --git a/app/Jobs/ProcessBillDetailDataJob.php b/app/Jobs/ProcessBillDetailDataJob.php index 5c373ad..fd7b724 100644 --- a/app/Jobs/ProcessBillDetailDataJob.php +++ b/app/Jobs/ProcessBillDetailDataJob.php @@ -16,7 +16,14 @@ { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; - protected string $period; + private const CSV_DELIMITER = '~'; + private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds + private const FILENAME = 'ST.AA.BILL.DETAILS.csv'; + private const DISK_NAME = 'sftpStatement'; + + private string $period; + private int $processedCount = 0; + private int $errorCount = 0; /** * Create a new job instance. @@ -30,76 +37,132 @@ * Execute the job. */ public function handle() + : void { try { - set_time_limit(24 * 60 * 60); - $disk = Storage::disk('sftpStatement'); - $processedCount = 0; - $errorCount = 0; + $this->initializeJob(); if ($this->period === '') { Log::warning('No period provided for bill detail data processing'); return; } - - // Construct the filename based on the period folder name - $filename = "{$this->period}.ST.AA.BILL.DETAILS.csv"; - $filePath = "{$this->period}/$filename"; - - Log::info("Processing bill detail file: $filePath"); - - if (!$disk->exists($filePath)) { - Log::warning("File not found: $filePath"); - return; - } - - // Create a temporary local copy of the file - $tempFilePath = storage_path("app/temp_$filename"); - file_put_contents($tempFilePath, $disk->get($filePath)); - - $handle = fopen($tempFilePath, "r"); - - if ($handle !== false) { - $headers = (new TempBillDetail())->getFillable(); - $rowCount = 0; - - while (($row = fgetcsv($handle, 0, "~")) !== false) { - $rowCount++; - - if (count($headers) === count($row)) { - $data = array_combine($headers, $row); - try { - if (isset($data['_id']) && $data['_id'] !== '_id') { - TempBillDetail::updateOrCreate( - ['_id' => $data['_id']], - $data - ); - $processedCount++; - } - } catch (Exception $e) { - $errorCount++; - Log::error("Error processing Bill Detail at row $rowCount in $filePath: " . $e->getMessage()); - } - } else { - Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row)); - } - } - - fclose($handle); - Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors."); - - // Clean up the temporary file - unlink($tempFilePath); - } else { - Log::error("Unable to open file: $filePath"); - } - - Log::info("Bill Detail data processing completed. Total processed: $processedCount, Total errors: $errorCount"); - + $this->processPeriod(); + $this->logJobCompletion(); } catch (Exception $e) { Log::error('Error in ProcessBillDetailDataJob: ' . $e->getMessage()); throw $e; } } + + private function initializeJob() + : void + { + set_time_limit(self::MAX_EXECUTION_TIME); + $this->processedCount = 0; + $this->errorCount = 0; + } + + private function processPeriod() + : void + { + $disk = Storage::disk(self::DISK_NAME); + $filename = "{$this->period}." . self::FILENAME; + $filePath = "{$this->period}/$filename"; + + if (!$this->validateFile($disk, $filePath)) { + return; + } + + $tempFilePath = $this->createTemporaryFile($disk, $filePath, $filename); + $this->processFile($tempFilePath, $filePath); + $this->cleanup($tempFilePath); + } + + private function validateFile($disk, string $filePath) + : bool + { + Log::info("Processing bill detail file: $filePath"); + + if (!$disk->exists($filePath)) { + Log::warning("File not found: $filePath"); + return false; + } + + return true; + } + + private function createTemporaryFile($disk, string $filePath, string $filename) + : string + { + $tempFilePath = storage_path("app/temp_$filename"); + file_put_contents($tempFilePath, $disk->get($filePath)); + return $tempFilePath; + } + + private function processFile(string $tempFilePath, string $filePath) + : void + { + $handle = fopen($tempFilePath, "r"); + if ($handle === false) { + Log::error("Unable to open file: $filePath"); + return; + } + + $headers = (new TempBillDetail())->getFillable(); + $rowCount = 0; + + while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) { + $rowCount++; + $this->processRow($row, $headers, $rowCount, $filePath); + } + + fclose($handle); + Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors."); + } + + private function processRow(array $row, array $headers, int $rowCount, string $filePath) + : void + { + if (count($headers) !== count($row)) { + Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . + count($headers) . ", Got: " . count($row)); + return; + } + + $data = array_combine($headers, $row); + $this->saveRecord($data, $rowCount, $filePath); + } + + private function saveRecord(array $data, int $rowCount, string $filePath) + : void + { + try { + if (isset($data['_id']) && $data['_id'] !== '_id') { + TempBillDetail::updateOrCreate( + ['_id' => $data['_id']], + $data + ); + $this->processedCount++; + } + } catch (Exception $e) { + $this->errorCount++; + Log::error("Error processing Bill Detail at row $rowCount in $filePath: " . $e->getMessage()); + } + } + + private function cleanup(string $tempFilePath) + : void + { + if (file_exists($tempFilePath)) { + unlink($tempFilePath); + } + } + + private function logJobCompletion() + : void + { + Log::info("Bill Detail data processing completed. " . + "Total processed: {$this->processedCount}, Total errors: {$this->errorCount}"); + } } diff --git a/app/Jobs/ProcessCategoryDataJob.php b/app/Jobs/ProcessCategoryDataJob.php index 3380984..a496c20 100644 --- a/app/Jobs/ProcessCategoryDataJob.php +++ b/app/Jobs/ProcessCategoryDataJob.php @@ -16,7 +16,26 @@ { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; - protected $period; + private const CSV_DELIMITER = '~'; + private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds + private const FILENAME = 'ST.CATEGORY.csv'; + private const DISK_NAME = 'sftpStatement'; + private const HEADER_MAP = [ + 'id' => 'id_category', + 'date_time' => 'date_time', + 'description' => 'description', + 'short_name' => 'short_name', + 'system_ind' => 'system_ind', + 'record_status' => 'record_status', + 'co_code' => 'co_code', + 'curr_no' => 'curr_no', + 'l_db_cr_ind' => 'l_db_cr_ind', + 'category_code' => 'category_code' + ]; + + private string $period; + private int $processedCount = 0; + private int $errorCount = 0; /** * Create a new job instance. @@ -33,98 +52,151 @@ : void { try { - set_time_limit(24 * 60 * 60); - $disk = Storage::disk('sftpStatement'); - $processedCount = 0; - $errorCount = 0; + $this->initializeJob(); if ($this->period === '') { Log::warning('No period provided for category data processing'); return; } - // Construct the filename based on the period folder name - $filename = "{$this->period}.ST.CATEGORY.csv"; - $filePath = "{$this->period}/$filename"; - - Log::info("Processing category file: $filePath"); - - if (!$disk->exists($filePath)) { - Log::warning("File not found: $filePath"); - return; - } - - // Create a temporary local copy of the file - $tempFilePath = storage_path("app/temp_$filename"); - file_put_contents($tempFilePath, $disk->get($filePath)); - - $handle = fopen($tempFilePath, "r"); - - if ($handle !== false) { - // Get the headers from the first row - $headerRow = fgetcsv($handle, 0, "~"); - - // Map the headers to our model fields - $headerMap = [ - 'id' => 'id_category', - 'date_time' => 'date_time', - 'description' => 'description', - 'short_name' => 'short_name', - 'system_ind' => 'system_ind', - 'record_status' => 'record_status', - 'co_code' => 'co_code', - 'curr_no' => 'curr_no', - 'l_db_cr_ind' => 'l_db_cr_ind', - 'category_code' => 'category_code' - ]; - - $rowCount = 0; - - while (($row = fgetcsv($handle, 0, "~")) !== false) { - $rowCount++; - - if (count($headerRow) === count($row)) { - // Combine the header row with the data row - $rawData = array_combine($headerRow, $row); - - // Map the raw data to our model fields - $data = []; - foreach ($headerMap as $csvField => $modelField) { - $data[$modelField] = $rawData[$csvField] ?? null; - } - - try { - // Skip header row if it was included in the data - if ($data['id_category'] !== 'id') { - // Use firstOrNew instead of updateOrCreate - $category = Category::firstOrNew(['id_category' => $data['id_category']]); - $category->fill($data); - $category->save(); - $processedCount++; - } - } catch (Exception $e) { - $errorCount++; - Log::error("Error processing Category at row $rowCount in $filePath: " . $e->getMessage()); - } - } else { - Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headerRow) . ", Got: " . count($row)); - } - } - - fclose($handle); - Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors."); - - // Clean up the temporary file - unlink($tempFilePath); - } else { - Log::error("Unable to open file: $filePath"); - } - - Log::info("Category data processing completed. Total processed: $processedCount, Total errors: $errorCount"); - + $this->processPeriod(); + $this->logJobCompletion(); } catch (Exception $e) { Log::error('Error in ProcessCategoryDataJob: ' . $e->getMessage()); throw $e; } } + + private function initializeJob() + : void + { + set_time_limit(self::MAX_EXECUTION_TIME); + $this->processedCount = 0; + $this->errorCount = 0; + } + + private function processPeriod() + : void + { + $disk = Storage::disk(self::DISK_NAME); + $filename = "{$this->period}." . self::FILENAME; + $filePath = "{$this->period}/$filename"; + + if (!$this->validateFile($disk, $filePath)) { + return; + } + + $tempFilePath = $this->createTemporaryFile($disk, $filePath, $filename); + $this->processFile($tempFilePath, $filePath); + $this->cleanup($tempFilePath); + } + + private function validateFile($disk, string $filePath) + : bool + { + Log::info("Processing category file: $filePath"); + + if (!$disk->exists($filePath)) { + Log::warning("File not found: $filePath"); + return false; + } + + return true; + } + + private function createTemporaryFile($disk, string $filePath, string $filename) + : string + { + $tempFilePath = storage_path("app/temp_$filename"); + file_put_contents($tempFilePath, $disk->get($filePath)); + return $tempFilePath; + } + + private function processFile(string $tempFilePath, string $filePath) + : void + { + $handle = fopen($tempFilePath, "r"); + if ($handle === false) { + Log::error("Unable to open file: $filePath"); + return; + } + + // Get the headers from the first row + $headerRow = fgetcsv($handle, 0, self::CSV_DELIMITER); + if (!$headerRow) { + fclose($handle); + return; + } + + $rowCount = 0; + while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) { + $rowCount++; + $this->processRow($headerRow, $row, $rowCount, $filePath); + } + + fclose($handle); + Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors."); + } + + private function processRow(array $headerRow, array $row, int $rowCount, string $filePath) + : void + { + if (count($headerRow) !== count($row)) { + Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . + count($headerRow) . ", Got: " . count($row)); + return; + } + + // Combine the header row with the data row + $rawData = array_combine($headerRow, $row); + $this->mapAndSaveRecord($rawData, $rowCount, $filePath); + } + + private function mapAndSaveRecord(array $rawData, int $rowCount, string $filePath) + : void + { + // Map the raw data to our model fields + $data = []; + foreach (self::HEADER_MAP as $csvField => $modelField) { + $data[$modelField] = $rawData[$csvField] ?? null; + } + + // Skip header row if it was included in the data + if ($data['id_category'] === 'id') { + return; + } + + $this->saveRecord($data, $rowCount, $filePath); + } + + private function saveRecord(array $data, int $rowCount, string $filePath) + : void + { + try { + // Use firstOrNew instead of updateOrCreate + $category = Category::firstOrNew(['id_category' => $data['id_category']]); + $category->fill($data); + $category->save(); + + $this->processedCount++; + } catch (Exception $e) { + $this->errorCount++; + Log::error("Error processing Category at row $rowCount in $filePath: " . $e->getMessage()); + } + } + + private function cleanup(string $tempFilePath) + : void + { + if (file_exists($tempFilePath)) { + unlink($tempFilePath); + } + } + + private function logJobCompletion() + : void + { + Log::info("Category data processing completed. " . + "Total processed: {$this->processedCount}, Total errors: {$this->errorCount}"); + } } diff --git a/app/Jobs/ProcessCompanyDataJob.php b/app/Jobs/ProcessCompanyDataJob.php index 6933976..5192a60 100644 --- a/app/Jobs/ProcessCompanyDataJob.php +++ b/app/Jobs/ProcessCompanyDataJob.php @@ -16,16 +16,38 @@ { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; - protected $period; - protected $filename; + private const CSV_DELIMITER = '~'; + private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds + private const FILENAME = 'ST.COMPANY.csv'; + private const DISK_NAME = 'sftpStatement'; + private const FIELD_MAP = [ + 'id' => null, // Not mapped to model + 'date_time' => null, // Not mapped to model + 'company_code' => 'code', + 'company_name' => 'name', + 'name_address' => 'address', + 'mnemonic' => 'mnemonic', + 'customer_company' => 'customer_company', + 'customer_mnemonic' => 'customer_mnemonic', + 'company_group' => 'company_group', + 'curr_no' => 'curr_no', + 'co_code' => 'co_code', + 'l_vendor_atm' => 'l_vendor_atm', + 'l_vendor_cpc' => 'l_vendor_cpc' + ]; + private const BOOLEAN_FIELDS = ['l_vendor_atm', 'l_vendor_cpc']; + + private string $period; + private string $filename; + private int $processedCount = 0; + private int $errorCount = 0; /** * Create a new job instance. */ - public function __construct(string $period = '', string $filename = "ST.COMPANY.csv") + public function __construct(string $period = '') { - $this->period = $period; - $this->filename = $filename; + $this->period = $period; } /** @@ -35,123 +57,154 @@ : void { try { - set_time_limit(24 * 60 * 60); - $disk = Storage::disk('sftpStatement'); - $processedCount = 0; - $errorCount = 0; + $this->initializeJob(); if ($this->period === '') { Log::warning('No period provided for company data processing'); return; } - // Construct the filepath based on the period folder name - $fileName = "{$this->period}.{$this->filename}"; - $filePath = "{$this->period}/$fileName"; - - Log::info("Processing company file: $filePath"); - - if (!$disk->exists($filePath)) { - Log::warning("File not found: $filePath"); - return; - } - - // Create a temporary local copy of the file - $tempFilePath = storage_path("app/temp_{$this->filename}"); - file_put_contents($tempFilePath, $disk->get($filePath)); - - $handle = fopen($tempFilePath, "r"); - - if ($handle !== false) { - // CSV headers from the file - $csvHeaders = [ - 'id', - 'date_time', - 'company_code', - 'company_name', - 'name_address', - 'mnemonic', - 'customer_company', - 'customer_mnemonic', - 'company_group', - 'curr_no', - 'co_code', - 'l_vendor_atm', - 'l_vendor_cpc' - ]; - - // Field mapping from CSV to Branch model - $fieldMapping = [ - 'company_code' => 'code', - 'company_name' => 'name', - 'name_address' => 'address', - 'mnemonic' => 'mnemonic', - 'customer_company' => 'customer_company', - 'customer_mnemonic' => 'customer_mnemonic', - 'company_group' => 'company_group', - 'curr_no' => 'curr_no', - 'co_code' => 'co_code', - 'l_vendor_atm' => 'l_vendor_atm', - 'l_vendor_cpc' => 'l_vendor_cpc' - ]; - - $rowCount = 0; - - while (($row = fgetcsv($handle, 0, "~")) !== false) { - $rowCount++; - - // Skip header row if it exists - if ($rowCount === 1 && (strtolower($row[0]) === 'id' || strtolower($row[2]) === 'company_code')) { - continue; - } - - if (count($csvHeaders) === count($row)) { - $csvData = array_combine($csvHeaders, $row); - - // Map CSV data to Branch model fields - $branchData = []; - foreach ($fieldMapping as $csvField => $modelField) { - if (isset($csvData[$csvField])) { - // Convert string boolean values to actual booleans for boolean fields - if (in_array($modelField, ['l_vendor_atm', 'l_vendor_cpc'])) { - $branchData[$modelField] = filter_var($csvData[$csvField], FILTER_VALIDATE_BOOLEAN); - } else { - $branchData[$modelField] = $csvData[$csvField]; - } - } - } - - try { - if (!empty($branchData['code'])) { - Branch::updateOrCreate( - ['code' => $branchData['code']], - $branchData - ); - $processedCount++; - } - } catch (Exception $e) { - $errorCount++; - Log::error("Error processing Company data at row $rowCount in $filePath: " . $e->getMessage()); - } - } else { - Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($csvHeaders) . ", Got: " . count($row)); - } - } - - fclose($handle); - Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors."); - - // Clean up the temporary file - unlink($tempFilePath); - } else { - Log::error("Unable to open file: $filePath"); - } - - Log::info("Company data processing completed. Total processed: $processedCount, Total errors: $errorCount"); - + $this->processPeriod(); + $this->logJobCompletion(); } catch (Exception $e) { Log::error('Error in ProcessCompanyDataJob: ' . $e->getMessage()); throw $e; } } + + private function initializeJob() + : void + { + set_time_limit(self::MAX_EXECUTION_TIME); + $this->processedCount = 0; + $this->errorCount = 0; + } + + private function processPeriod() + : void + { + $disk = Storage::disk(self::DISK_NAME); + $filename = "{$this->period}." . self::FILENAME; + $filePath = "{$this->period}/$filename"; + + if (!$this->validateFile($disk, $filePath)) { + return; + } + + $tempFilePath = $this->createTemporaryFile($disk, $filePath, $filename); + $this->processFile($tempFilePath, $filePath); + $this->cleanup($tempFilePath); + } + + private function validateFile($disk, string $filePath) + : bool + { + Log::info("Processing company file: $filePath"); + + if (!$disk->exists($filePath)) { + Log::warning("File not found: $filePath"); + return false; + } + + return true; + } + + private function createTemporaryFile($disk, string $filePath, string $filename = self::FILENAME) + : string + { + $tempFilePath = storage_path("app/temp_$filename"); + file_put_contents($tempFilePath, $disk->get($filePath)); + return $tempFilePath; + } + + private function processFile(string $tempFilePath, string $filePath) + : void + { + $handle = fopen($tempFilePath, "r"); + if ($handle === false) { + Log::error("Unable to open file: $filePath"); + return; + } + + $rowCount = 0; + while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) { + $rowCount++; + + // Skip header row if it exists + if ($rowCount === 1 && (strtolower($row[0]) === 'id' || strtolower($row[2]) === 'company_code')) { + continue; + } + + $this->processRow($row, $rowCount, $filePath); + } + + fclose($handle); + Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors."); + } + + private function processRow(array $row, int $rowCount, string $filePath) + : void + { + $csvHeaders = array_keys(self::FIELD_MAP); + + if (count($csvHeaders) !== count($row)) { + Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . + count($csvHeaders) . ", Got: " . count($row)); + return; + } + + $csvData = array_combine($csvHeaders, $row); + $this->mapAndSaveRecord($csvData, $rowCount, $filePath); + } + + private function mapAndSaveRecord(array $csvData, int $rowCount, string $filePath) + : void + { + // Map CSV data to Branch model fields + $branchData = []; + foreach (self::FIELD_MAP as $csvField => $modelField) { + if ($modelField !== null && isset($csvData[$csvField])) { + // Convert string boolean values to actual booleans for boolean fields + if (in_array($modelField, self::BOOLEAN_FIELDS)) { + $branchData[$modelField] = filter_var($csvData[$csvField], FILTER_VALIDATE_BOOLEAN); + } else { + $branchData[$modelField] = $csvData[$csvField]; + } + } + } + + $this->saveRecord($branchData, $rowCount, $filePath); + } + + private function saveRecord(array $branchData, int $rowCount, string $filePath) + : void + { + try { + if (!empty($branchData['code'])) { + Branch::updateOrCreate( + ['code' => $branchData['code']], + $branchData + ); + $this->processedCount++; + } + } catch (Exception $e) { + $this->errorCount++; + Log::error("Error processing Company data at row $rowCount in $filePath: " . $e->getMessage()); + } + } + + private function cleanup(string $tempFilePath) + : void + { + if (file_exists($tempFilePath)) { + unlink($tempFilePath); + } + } + + private function logJobCompletion() + : void + { + Log::info("Company data processing completed. " . + "Total processed: {$this->processedCount}, Total errors: {$this->errorCount}"); + } } diff --git a/app/Jobs/ProcessCustomerDataJob.php b/app/Jobs/ProcessCustomerDataJob.php index 173c8db..8d7475c 100644 --- a/app/Jobs/ProcessCustomerDataJob.php +++ b/app/Jobs/ProcessCustomerDataJob.php @@ -16,82 +16,152 @@ { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; - protected $period; + private const CSV_DELIMITER = '~'; + private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds + private const FILENAME = 'ST.CUSTOMER.csv'; + private const DISK_NAME = 'sftpStatement'; + private string $period; + private int $processedCount = 0; + private int $errorCount = 0; + + /** + * Create a new job instance. + */ public function __construct(string $period = '') { $this->period = $period; } + /** + * Execute the job. + */ public function handle() + : void { try { - set_time_limit(24 * 60 * 60); - $disk = Storage::disk('sftpStatement'); - $processedCount = 0; - $errorCount = 0; + $this->initializeJob(); if ($this->period === '') { Log::warning('No period provided for customer data processing'); return; } - // Construct the filename based on the period folder name - $filename = "{$this->period}.ST.CUSTOMER.csv"; - $filePath = "{$this->period}/$filename"; - - Log::info("Processing customer file: $filePath"); - - if (!$disk->exists($filePath)) { - Log::warning("File not found: $filePath"); - return; - } - - // Create a temporary local copy of the file - $tempFilePath = storage_path("app/temp_$filename"); - file_put_contents($tempFilePath, $disk->get($filePath)); - - $handle = fopen($tempFilePath, "r"); - - if ($handle !== false) { - $headers = (new Customer())->getFillable(); - $rowCount = 0; - - while (($row = fgetcsv($handle, 0, "~")) !== false) { - $rowCount++; - - if (count($headers) === count($row)) { - $data = array_combine($headers, $row); - try { - if ($data['customer_code'] !== 'customer_code') { - $customer = Customer::firstOrNew(['customer_code' => $data['customer_code']]); - $customer->fill($data); - $customer->save(); - $processedCount++; - } - } catch (Exception $e) { - $errorCount++; - Log::error("Error processing Customer at row $rowCount in $filePath: " . $e->getMessage()); - } - } else { - Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row)); - } - } - - fclose($handle); - Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors."); - - // Clean up the temporary file - unlink($tempFilePath); - } else { - Log::error("Unable to open file: $filePath"); - } - - Log::info("Customer data processing completed. Total processed: $processedCount, Total errors: $errorCount"); - + $this->processPeriod(); + $this->logJobCompletion(); } catch (Exception $e) { Log::error('Error in ProcessCustomerDataJob: ' . $e->getMessage()); throw $e; } } + + private function initializeJob() + : void + { + set_time_limit(self::MAX_EXECUTION_TIME); + $this->processedCount = 0; + $this->errorCount = 0; + } + + private function processPeriod() + : void + { + $disk = Storage::disk(self::DISK_NAME); + $filename = "{$this->period}." . self::FILENAME; + $filePath = "{$this->period}/$filename"; + + if (!$this->validateFile($disk, $filePath)) { + return; + } + + $tempFilePath = $this->createTemporaryFile($disk, $filePath, $filename); + $this->processFile($tempFilePath, $filePath); + $this->cleanup($tempFilePath); + } + + private function validateFile($disk, string $filePath) + : bool + { + Log::info("Processing customer file: $filePath"); + + if (!$disk->exists($filePath)) { + Log::warning("File not found: $filePath"); + return false; + } + + return true; + } + + private function createTemporaryFile($disk, string $filePath, string $filename) + : string + { + $tempFilePath = storage_path("app/temp_$filename"); + file_put_contents($tempFilePath, $disk->get($filePath)); + return $tempFilePath; + } + + private function processFile(string $tempFilePath, string $filePath) + : void + { + $handle = fopen($tempFilePath, "r"); + if ($handle === false) { + Log::error("Unable to open file: $filePath"); + return; + } + + $headers = (new Customer())->getFillable(); + $rowCount = 0; + + while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) { + $rowCount++; + $this->processRow($row, $headers, $rowCount, $filePath); + } + + fclose($handle); + Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors."); + } + + private function processRow(array $row, array $headers, int $rowCount, string $filePath) + : void + { + if (count($headers) !== count($row)) { + Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . + count($headers) . ", Got: " . count($row)); + return; + } + + $data = array_combine($headers, $row); + $this->saveRecord($data, $rowCount, $filePath); + } + + private function saveRecord(array $data, int $rowCount, string $filePath) + : void + { + try { + if (isset($data['customer_code']) && $data['customer_code'] !== 'customer_code') { + $customer = Customer::firstOrNew(['customer_code' => $data['customer_code']]); + $customer->fill($data); + $customer->save(); + $this->processedCount++; + } + } catch (Exception $e) { + $this->errorCount++; + Log::error("Error processing Customer at row $rowCount in $filePath: " . $e->getMessage()); + } + } + + private function cleanup(string $tempFilePath) + : void + { + if (file_exists($tempFilePath)) { + unlink($tempFilePath); + } + } + + private function logJobCompletion() + : void + { + Log::info("Customer data processing completed. " . + "Total processed: {$this->processedCount}, Total errors: {$this->errorCount}"); + } } diff --git a/app/Jobs/ProcessDataCaptureDataJob.php b/app/Jobs/ProcessDataCaptureDataJob.php index 3e715c8..700a957 100644 --- a/app/Jobs/ProcessDataCaptureDataJob.php +++ b/app/Jobs/ProcessDataCaptureDataJob.php @@ -16,16 +16,63 @@ { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; - protected $period; - protected $filename; + private const CSV_DELIMITER = '~'; + private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds + private const FILENAME = 'ST.DATA.CAPTURE.csv'; + private const DISK_NAME = 'sftpStatement'; + private const CSV_HEADERS = [ + 'id', + 'account_number', + 'sign', + 'amount_lcy', + 'transaction_code', + 'their_reference', + 'narrative', + 'pl_category', + 'customer_id', + 'account_officer', + 'product_category', + 'value_date', + 'currency', + 'amount_fcy', + 'exchange_rate', + 'neg_ref_no', + 'position_type', + 'our_reference', + 'reversal_marker', + 'exposure_date', + 'currency_market', + 'iblc_country', + 'last_version', + 'otor_version', + 'department_code', + 'dealer_desk', + 'bank_sort_cde', + 'cheque_number', + 'accounting_date', + 'contingent_acct', + 'cheq_type', + 'tfs_reference', + 'accounting_company', + 'stmt_no', + 'curr_no', + 'inputter', + 'authoriser', + 'co_code', + 'date_time' + ]; + private const DATE_FIELDS = ['value_date', 'exposure_date', 'accounting_date']; + + private string $period; + private int $processedCount = 0; + private int $errorCount = 0; /** * Create a new job instance. */ - public function __construct(string $period = '', string $filename = "ST.DATA.CAPTURE.csv") + public function __construct(string $period = '') { - $this->period = $period; - $this->filename = $filename; + $this->period = $period; } /** @@ -35,143 +82,160 @@ : void { try { - set_time_limit(24 * 60 * 60); - $disk = Storage::disk('sftpStatement'); - $processedCount = 0; - $errorCount = 0; + $this->initializeJob(); if ($this->period === '') { Log::warning('No period provided for data capture processing'); return; } - // Construct the filepath based on the period folder name - $fileName = "{$this->period}.{$this->filename}"; - $filePath = "{$this->period}/$fileName"; - - Log::info("Processing data capture file: $filePath"); - - if (!$disk->exists($filePath)) { - Log::warning("File not found: $filePath"); - return; - } - - // Create a temporary local copy of the file - $tempFilePath = storage_path("app/temp_{$this->filename}"); - file_put_contents($tempFilePath, $disk->get($filePath)); - - $handle = fopen($tempFilePath, "r"); - - if ($handle !== false) { - // CSV headers from the file - $csvHeaders = [ - 'id', - 'account_number', - 'sign', - 'amount_lcy', - 'transaction_code', - 'their_reference', - 'narrative', - 'pl_category', - 'customer_id', - 'account_officer', - 'product_category', - 'value_date', - 'currency', - 'amount_fcy', - 'exchange_rate', - 'neg_ref_no', - 'position_type', - 'our_reference', - 'reversal_marker', - 'exposure_date', - 'currency_market', - 'iblc_country', - 'last_version', - 'otor_version', - 'department_code', - 'dealer_desk', - 'bank_sort_cde', - 'cheque_number', - 'accounting_date', - 'contingent_acct', - 'cheq_type', - 'tfs_reference', - 'accounting_company', - 'stmt_no', - 'curr_no', - 'inputter', - 'authoriser', - 'co_code', - 'date_time' - ]; - - $rowCount = 0; - - while (($row = fgetcsv($handle, 0, "~")) !== false) { - $rowCount++; - - // Skip header row if it exists - if ($rowCount === 1 && strtolower($row[0]) === 'id') { - continue; - } - - if (count($csvHeaders) === count($row)) { - $data = array_combine($csvHeaders, $row); - - try { - // Format dates if they exist - foreach (['value_date', 'exposure_date', 'accounting_date'] as $dateField) { - if (!empty($data[$dateField])) { - try { - $data[$dateField] = date('Y-m-d', strtotime($data[$dateField])); - } catch (Exception $e) { - // If date parsing fails, keep the original value - Log::warning("Failed to parse date for $dateField: {$data[$dateField]}"); - } - } - } - - // Format datetime if it exists - if (!empty($data['date_time'])) { - try { - $data['date_time'] = date('Y-m-d H:i:s', strtotime($data['date_time'])); - } catch (Exception $e) { - // If datetime parsing fails, keep the original value - Log::warning("Failed to parse datetime for date_time: {$data['date_time']}"); - } - } - - if (!empty($data['id'])) { - DataCapture::updateOrCreate( - ['id' => $data['id']], - $data - ); - $processedCount++; - } - } catch (Exception $e) { - $errorCount++; - Log::error("Error processing Data Capture at row $rowCount in $filePath: " . $e->getMessage()); - } - } else { - Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($csvHeaders) . ", Got: " . count($row)); - } - } - - fclose($handle); - Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors."); - - // Clean up the temporary file - unlink($tempFilePath); - } else { - Log::error("Unable to open file: $filePath"); - } - - Log::info("Data capture processing completed. Total processed: $processedCount, Total errors: $errorCount"); - + $this->processPeriod(); + $this->logJobCompletion(); } catch (Exception $e) { Log::error('Error in ProcessDataCaptureDataJob: ' . $e->getMessage()); throw $e; } } + + private function initializeJob() + : void + { + set_time_limit(self::MAX_EXECUTION_TIME); + $this->processedCount = 0; + $this->errorCount = 0; + } + + private function processPeriod() + : void + { + $disk = Storage::disk(self::DISK_NAME); + $filename = "{$this->period}." . self::FILENAME; + $filePath = "{$this->period}/$filename"; + + if (!$this->validateFile($disk, $filePath)) { + return; + } + + $tempFilePath = $this->createTemporaryFile($disk, $filePath, $filename); + $this->processFile($tempFilePath, $filePath); + $this->cleanup($tempFilePath); + } + + private function validateFile($disk, string $filePath) + : bool + { + Log::info("Processing data capture file: $filePath"); + + if (!$disk->exists($filePath)) { + Log::warning("File not found: $filePath"); + return false; + } + + return true; + } + + private function createTemporaryFile($disk, string $filePath, string $filename) + : string + { + $tempFilePath = storage_path("app/temp_$filename"); + file_put_contents($tempFilePath, $disk->get($filePath)); + return $tempFilePath; + } + + private function processFile(string $tempFilePath, string $filePath) + : void + { + $handle = fopen($tempFilePath, "r"); + if ($handle === false) { + Log::error("Unable to open file: $filePath"); + return; + } + + $rowCount = 0; + while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) { + $rowCount++; + + // Skip header row if it exists + if ($rowCount === 1 && strtolower($row[0]) === 'id') { + continue; + } + + $this->processRow($row, $rowCount, $filePath); + } + + fclose($handle); + Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors."); + } + + private function processRow(array $row, int $rowCount, string $filePath) + : void + { + if (count(self::CSV_HEADERS) !== count($row)) { + Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . + count(self::CSV_HEADERS) . ", Got: " . count($row)); + return; + } + + $data = array_combine(self::CSV_HEADERS, $row); + $this->formatDates($data); + $this->saveRecord($data, $rowCount, $filePath); + } + + private function formatDates(array &$data) + : void + { + // Format dates if they exist + foreach (self::DATE_FIELDS as $dateField) { + if (!empty($data[$dateField])) { + try { + $data[$dateField] = date('Y-m-d', strtotime($data[$dateField])); + } catch (Exception $e) { + // If date parsing fails, keep the original value + Log::warning("Failed to parse date for $dateField: {$data[$dateField]}"); + } + } + } + + // Format datetime if it exists + if (!empty($data['date_time'])) { + try { + $data['date_time'] = date('Y-m-d H:i:s', strtotime($data['date_time'])); + } catch (Exception $e) { + // If datetime parsing fails, keep the original value + Log::warning("Failed to parse datetime for date_time: {$data['date_time']}"); + } + } + } + + private function saveRecord(array $data, int $rowCount, string $filePath) + : void + { + try { + if (!empty($data['id'])) { + DataCapture::updateOrCreate( + ['id' => $data['id']], + $data + ); + $this->processedCount++; + } + } catch (Exception $e) { + $this->errorCount++; + Log::error("Error processing Data Capture at row $rowCount in $filePath: " . $e->getMessage()); + } + } + + private function cleanup(string $tempFilePath) + : void + { + if (file_exists($tempFilePath)) { + unlink($tempFilePath); + } + } + + private function logJobCompletion() + : void + { + Log::info("Data capture processing completed. " . + "Total processed: {$this->processedCount}, Total errors: {$this->errorCount}"); + } } diff --git a/app/Jobs/ProcessFtTxnTypeConditionJob.php b/app/Jobs/ProcessFtTxnTypeConditionJob.php index ceefbcb..caca973 100644 --- a/app/Jobs/ProcessFtTxnTypeConditionJob.php +++ b/app/Jobs/ProcessFtTxnTypeConditionJob.php @@ -1,113 +1,175 @@ periods = is_string($periods) ? [$periods] : $periods; - } + use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; + + private const CSV_DELIMITER = '~'; + private const EXPECTED_HEADERS = [ + 'id', + 'date_time', + 'transaction_type', + 'short_descr', + 'txn_code_cr', + 'txn_code_dr' + ]; + private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds + private const FILENAME = 'ST.FT.TXN.TYPE.CONDITION.csv'; + private const DISK_NAME = 'sftpStatement'; + + private string $period; + private int $processedCount = 0; + private int $errorCount = 0; + + public function __construct(string $period = '') + { + $this->period = $period; + } + + public function handle() + : void + { + try { + $this->initializeJob(); + + if ($this->period === '') { + Log::warning('No periods provided for transaction type condition data processing'); + return; + } + + $this->processPeriod(); + $this->logJobCompletion(); + } catch (Exception $e) { + Log::error('Error in ProcessFtTxnTypeConditionJob: ' . $e->getMessage()); + throw $e; + } + } + + private function initializeJob() + : void + { + set_time_limit(self::MAX_EXECUTION_TIME); + $this->processedCount = 0; + $this->errorCount = 0; + } + + private function processPeriod() + : void + { + $disk = Storage::disk(self::DISK_NAME); + $filePath = "$this->period/" . self::FILENAME; - /** - * Execute the job. - */ - public function handle(): void - { - try { - set_time_limit(24 * 60 * 60); - $disk = Storage::disk('sftpStatement'); - $processedCount = 0; - $errorCount = 0; - - if (empty($this->periods)) { - Log::warning('No periods provided for transaction type condition data processing'); + if (!$this->validateFile($disk, $filePath)) { return; } - foreach ($this->periods as $period) { + $tempFilePath = $this->createTemporaryFile($disk, $filePath); + $this->processFile($tempFilePath, $filePath); + $this->cleanup($tempFilePath); + } - // Construct the filepath based on the period folder name - $filePath = "$period/{$this->filename}"; + private function validateFile($disk, string $filePath) + : bool + { + Log::info("Processing transaction type condition file: $filePath"); - Log::info("Processing transaction type condition file: $filePath"); + if (!$disk->exists($filePath)) { + Log::warning("File not found: $filePath"); + return false; + } - if (!$disk->exists($filePath)) { - Log::warning("File not found: $filePath"); + return true; + } + + private function createTemporaryFile($disk, string $filePath) + : string + { + $tempFilePath = storage_path("app/temp_" . self::FILENAME); + file_put_contents($tempFilePath, $disk->get($filePath)); + return $tempFilePath; + } + + private function processFile(string $tempFilePath, string $filePath) + : void + { + $handle = fopen($tempFilePath, "r"); + if ($handle === false) { + Log::error("Unable to open file: $filePath"); + return; + } + + $rowCount = 0; + while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) { + $rowCount++; + if ($this->isHeaderRow($rowCount, $row)) { continue; } - // Create a temporary local copy of the file - $tempFilePath = storage_path("app/temp_{$this->filename}"); - file_put_contents($tempFilePath, $disk->get($filePath)); - - $handle = fopen($tempFilePath, "r"); - - if ($handle !== false) { - $headers = ['id', 'date_time', 'transaction_type', 'short_descr', 'txn_code_cr', 'txn_code_dr']; - $rowCount = 0; - - while (($row = fgetcsv($handle, 0, "~")) !== false) { - $rowCount++; - - // Skip header row if it exists - if ($rowCount === 1 && strtolower($row[0]) === 'id') { - continue; - } - - if (count($headers) === count($row)) { - $data = array_combine($headers, $row); - try { - if (!empty($data['id'])) { - FtTxnTypeCondition::updateOrCreate( - ['id' => $data['id']], - $data - ); - $processedCount++; - } - } catch (Exception $e) { - $errorCount++; - Log::error("Error processing Transaction Type Condition at row $rowCount in $filePath: " . $e->getMessage()); - } - } else { - Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row)); - } - } - - fclose($handle); - Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors."); - - // Clean up the temporary file - unlink($tempFilePath); - } else { - Log::error("Unable to open file: $filePath"); - } + $this->processRow($row, $rowCount, $filePath); } - Log::info("Transaction type condition data processing completed. Total processed: $processedCount, Total errors: $errorCount"); + fclose($handle); + Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors."); + } - } catch (Exception $e) { - Log::error('Error in ProcessFtTxnTypeConditionJob: ' . $e->getMessage()); - throw $e; + private function isHeaderRow(int $rowCount, array $row) + : bool + { + return $rowCount === 1 && strtolower($row[0]) === 'id'; + } + + private function processRow(array $row, int $rowCount, string $filePath) + : void + { + if (count(self::EXPECTED_HEADERS) !== count($row)) { + Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . + count(self::EXPECTED_HEADERS) . ", Got: " . count($row)); + return; + } + + $data = array_combine(self::EXPECTED_HEADERS, $row); + $this->saveRecord($data, $rowCount, $filePath); + } + + private function saveRecord(array $data, int $rowCount, string $filePath) + : void + { + try { + if (!empty($data['id'])) { + FtTxnTypeCondition::updateOrCreate(['id' => $data['id']], $data); + $this->processedCount++; + } + } catch (Exception $e) { + $this->errorCount++; + Log::error("Error processing Transaction Type Condition at row $rowCount in $filePath: " . $e->getMessage()); + } + } + + private function cleanup(string $tempFilePath) + : void + { + if (file_exists($tempFilePath)) { + unlink($tempFilePath); + } + } + + private function logJobCompletion() + : void + { + Log::info("Transaction type condition data processing completed. " . + "Total processed: {$this->processedCount}, Total errors: {$this->errorCount}"); } } -} diff --git a/app/Jobs/ProcessFundsTransferDataJob.php b/app/Jobs/ProcessFundsTransferDataJob.php index 1b63f14..6ef5ede 100644 --- a/app/Jobs/ProcessFundsTransferDataJob.php +++ b/app/Jobs/ProcessFundsTransferDataJob.php @@ -16,7 +16,14 @@ { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; - protected $period; + private const CSV_DELIMITER = '~'; + private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds + private const FILENAME = 'ST.FUNDS.TRANSFER.csv'; + private const DISK_NAME = 'sftpStatement'; + + private string $period; + private int $processedCount = 0; + private int $errorCount = 0; /** * Create a new job instance. @@ -33,78 +40,134 @@ : void { try { - set_time_limit(24 * 60 * 60); - $disk = Storage::disk('sftpStatement'); - $processedCount = 0; - $errorCount = 0; + $this->initializeJob(); if ($this->period === '') { Log::warning('No period provided for funds transfer data processing'); return; } - // Construct the filename based on the period folder name - $filename = "{$this->period}.ST.FUNDS.TRANSFER.csv"; - $filePath = "{$this->period}/$filename"; - - Log::info("Processing funds transfer file: $filePath"); - - if (!$disk->exists($filePath)) { - Log::warning("File not found: $filePath"); - return; - } - - // Create a temporary local copy of the file - $tempFilePath = storage_path("app/temp_$filename"); - file_put_contents($tempFilePath, $disk->get($filePath)); - - $handle = fopen($tempFilePath, "r"); - - if ($handle !== false) { - $headers = (new TempFundsTransfer())->getFillable(); - $rowCount = 0; - - while (($row = fgetcsv($handle, 0, "~")) !== false) { - $rowCount++; - - // Handle case where row has more columns than headers - if (count($row) > count($headers)) { - $row = array_slice($row, 0, count($headers)); - } - - if (count($headers) === count($row)) { - $data = array_combine($headers, $row); - try { - if (isset($data['_id']) && $data['_id'] !== '_id') { - TempFundsTransfer::updateOrCreate( - ['_id' => $data['_id']], - $data - ); - $processedCount++; - } - } catch (Exception $e) { - $errorCount++; - Log::error("Error processing Funds Transfer at row $rowCount in $filePath: " . $e->getMessage()); - } - } else { - Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row)); - } - } - - fclose($handle); - Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors."); - - // Clean up the temporary file - unlink($tempFilePath); - } else { - Log::error("Unable to open file: $filePath"); - } - - Log::info("Funds Transfer data processing completed. Total processed: $processedCount, Total errors: $errorCount"); - + $this->processPeriod(); + $this->logJobCompletion(); } catch (Exception $e) { Log::error('Error in ProcessFundsTransferDataJob: ' . $e->getMessage()); throw $e; } } + + private function initializeJob() + : void + { + set_time_limit(self::MAX_EXECUTION_TIME); + $this->processedCount = 0; + $this->errorCount = 0; + } + + private function processPeriod() + : void + { + $disk = Storage::disk(self::DISK_NAME); + $filename = "{$this->period}." . self::FILENAME; + $filePath = "{$this->period}/$filename"; + + if (!$this->validateFile($disk, $filePath)) { + return; + } + + $tempFilePath = $this->createTemporaryFile($disk, $filePath, $filename); + $this->processFile($tempFilePath, $filePath); + $this->cleanup($tempFilePath); + } + + private function validateFile($disk, string $filePath) + : bool + { + Log::info("Processing funds transfer file: $filePath"); + + if (!$disk->exists($filePath)) { + Log::warning("File not found: $filePath"); + return false; + } + + return true; + } + + private function createTemporaryFile($disk, string $filePath, string $filename) + : string + { + $tempFilePath = storage_path("app/temp_$filename"); + file_put_contents($tempFilePath, $disk->get($filePath)); + return $tempFilePath; + } + + private function processFile(string $tempFilePath, string $filePath) + : void + { + $handle = fopen($tempFilePath, "r"); + if ($handle === false) { + Log::error("Unable to open file: $filePath"); + return; + } + + $headers = (new TempFundsTransfer())->getFillable(); + $rowCount = 0; + + while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) { + $rowCount++; + $this->processRow($row, $headers, $rowCount, $filePath); + } + + fclose($handle); + Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors."); + } + + private function processRow(array $row, array $headers, int $rowCount, string $filePath) + : void + { + // Handle case where row has more columns than headers + if (count($row) > count($headers)) { + $row = array_slice($row, 0, count($headers)); + } + + if (count($headers) !== count($row)) { + Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . + count($headers) . ", Got: " . count($row)); + return; + } + + $data = array_combine($headers, $row); + $this->saveRecord($data, $rowCount, $filePath); + } + + private function saveRecord(array $data, int $rowCount, string $filePath) + : void + { + try { + if (isset($data['_id']) && $data['_id'] !== '_id') { + TempFundsTransfer::updateOrCreate( + ['_id' => $data['_id']], + $data + ); + $this->processedCount++; + } + } catch (Exception $e) { + $this->errorCount++; + Log::error("Error processing Funds Transfer at row $rowCount in $filePath: " . $e->getMessage()); + } + } + + private function cleanup(string $tempFilePath) + : void + { + if (file_exists($tempFilePath)) { + unlink($tempFilePath); + } + } + + private function logJobCompletion() + : void + { + Log::info("Funds Transfer data processing completed. " . + "Total processed: {$this->processedCount}, Total errors: {$this->errorCount}"); + } } diff --git a/app/Jobs/ProcessStmtEntryDataJob.php b/app/Jobs/ProcessStmtEntryDataJob.php index 6621f7c..e6881e6 100644 --- a/app/Jobs/ProcessStmtEntryDataJob.php +++ b/app/Jobs/ProcessStmtEntryDataJob.php @@ -16,7 +16,14 @@ { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; - protected $period; + private const CSV_DELIMITER = '~'; + private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds + private const FILENAME = 'ST.STMT.ENTRY.csv'; + private const DISK_NAME = 'sftpStatement'; + + private string $period; + private int $processedCount = 0; + private int $errorCount = 0; /** * Create a new job instance. @@ -33,84 +40,139 @@ : void { try { - set_time_limit(24 * 60 * 60); - $disk = Storage::disk('sftpStatement'); - $processedCount = 0; - $errorCount = 0; + $this->initializeJob(); if ($this->period === '') { Log::warning('No period provided for statement entry data processing'); return; } - // Construct the filename based on the period folder name - $filename = "{$this->period}.ST.STMT.ENTRY.csv"; - $filePath = "{$this->period}/$filename"; - - Log::info("Processing statement entry file: $filePath"); - - if (!$disk->exists($filePath)) { - Log::warning("File not found: $filePath"); - return; - } - - // Create a temporary local copy of the file - $tempFilePath = storage_path("app/temp_$filename"); - file_put_contents($tempFilePath, $disk->get($filePath)); - - $handle = fopen($tempFilePath, "r"); - - if ($handle !== false) { - $headers = (new StmtEntry())->getFillable(); - $rowCount = 0; - - while (($row = fgetcsv($handle, 0, "~")) !== false) { - $rowCount++; - - if (count($headers) === count($row)) { - $data = array_combine($headers, $row); - try { - if ($data['stmt_entry_id'] !== 'stmt_entry_id') { - // Bersihkan trans_reference dari \\BNK jika ada - if (isset($data['trans_reference'])) { - $cleanedRef = preg_replace('/\\\\.*$/', '', $data['trans_reference']); - //$firstTwoChars = substr($cleanedRef, 0, 2); - /*$data['trans_reference'] = (in_array($firstTwoChars, ['FT', 'TT','AA'])) - ? substr($cleanedRef, 0, 12) - : $cleanedRef;*/ - - $data['trans_reference'] = $cleanedRef; - } - - StmtEntry::updateOrCreate( - ['stmt_entry_id' => $data['stmt_entry_id']], - $data - ); - $processedCount++; - } - } catch (Exception $e) { - $errorCount++; - Log::error("Error processing Statement Entry at row $rowCount in $filePath: " . $e->getMessage()); - } - } else { - Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row)); - } - } - - fclose($handle); - Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors."); - - // Clean up the temporary file - unlink($tempFilePath); - } else { - Log::error("Unable to open file: $filePath"); - } - - Log::info("Statement Entry data processing completed. Total processed: $processedCount, Total errors: $errorCount"); - + $this->processPeriod(); + $this->logJobCompletion(); } catch (Exception $e) { Log::error('Error in ProcessStmtEntryDataJob: ' . $e->getMessage()); throw $e; } } + + private function initializeJob() + : void + { + set_time_limit(self::MAX_EXECUTION_TIME); + $this->processedCount = 0; + $this->errorCount = 0; + } + + private function processPeriod() + : void + { + $disk = Storage::disk(self::DISK_NAME); + $filename = "{$this->period}." . self::FILENAME; + $filePath = "{$this->period}/$filename"; + + if (!$this->validateFile($disk, $filePath)) { + return; + } + + $tempFilePath = $this->createTemporaryFile($disk, $filePath, $filename); + $this->processFile($tempFilePath, $filePath); + $this->cleanup($tempFilePath); + } + + private function validateFile($disk, string $filePath) + : bool + { + Log::info("Processing statement entry file: $filePath"); + + if (!$disk->exists($filePath)) { + Log::warning("File not found: $filePath"); + return false; + } + + return true; + } + + private function createTemporaryFile($disk, string $filePath, string $filename) + : string + { + $tempFilePath = storage_path("app/temp_$filename"); + file_put_contents($tempFilePath, $disk->get($filePath)); + return $tempFilePath; + } + + private function processFile(string $tempFilePath, string $filePath) + : void + { + $handle = fopen($tempFilePath, "r"); + if ($handle === false) { + Log::error("Unable to open file: $filePath"); + return; + } + + $headers = (new StmtEntry())->getFillable(); + $rowCount = 0; + + while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) { + $rowCount++; + $this->processRow($row, $headers, $rowCount, $filePath); + } + + fclose($handle); + Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors."); + } + + private function processRow(array $row, array $headers, int $rowCount, string $filePath) + : void + { + if (count($headers) !== count($row)) { + Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . + count($headers) . ", Got: " . count($row)); + return; + } + + $data = array_combine($headers, $row); + $this->cleanTransReference($data); + $this->saveRecord($data, $rowCount, $filePath); + } + + private function cleanTransReference(array &$data) + : void + { + if (isset($data['trans_reference'])) { + // Clean trans_reference from \\BNK if present + $data['trans_reference'] = preg_replace('/\\\\.*$/', '', $data['trans_reference']); + } + } + + private function saveRecord(array $data, int $rowCount, string $filePath) + : void + { + try { + if (isset($data['stmt_entry_id']) && $data['stmt_entry_id'] !== 'stmt_entry_id') { + StmtEntry::updateOrCreate( + ['stmt_entry_id' => $data['stmt_entry_id']], + $data + ); + $this->processedCount++; + } + } catch (Exception $e) { + $this->errorCount++; + Log::error("Error processing Statement Entry at row $rowCount in $filePath: " . $e->getMessage()); + } + } + + private function cleanup(string $tempFilePath) + : void + { + if (file_exists($tempFilePath)) { + unlink($tempFilePath); + } + } + + private function logJobCompletion() + : void + { + Log::info("Statement Entry data processing completed. " . + "Total processed: {$this->processedCount}, Total errors: {$this->errorCount}"); + } } diff --git a/app/Jobs/ProcessStmtNarrFormatDataJob.php b/app/Jobs/ProcessStmtNarrFormatDataJob.php index 7aae0f6..04b3cbb 100644 --- a/app/Jobs/ProcessStmtNarrFormatDataJob.php +++ b/app/Jobs/ProcessStmtNarrFormatDataJob.php @@ -1,113 +1,164 @@ periods = is_string($periods) ? [$periods] : $periods; - } + use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; + private const CSV_DELIMITER = '~'; + private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds + private const FILENAME = 'ST.STMT.NARR.FORMAT.csv'; + private const DISK_NAME = 'sftpStatement'; - /** - * Execute the job. - */ - public function handle(): void - { - try { - set_time_limit(24 * 60 * 60); - $disk = Storage::disk('sftpStatement'); - $processedCount = 0; - $errorCount = 0; + private string $period; + private int $processedCount = 0; + private int $errorCount = 0; - if (empty($this->periods)) { - Log::warning('No periods provided for statement narrative format data processing'); + /** + * Create a new job instance. + */ + public function __construct(string $period = '') + { + $this->period = $period; + } + + /** + * Execute the job. + */ + public function handle() + : void + { + try { + $this->initializeJob(); + + if ($this->period === '') { + Log::warning('No periods provided for statement narrative format data processing'); + return; + } + + $this->processPeriod(); + $this->logJobCompletion(); + } catch (Exception $e) { + Log::error('Error in ProcessStmtNarrFormatDataJob: ' . $e->getMessage()); + throw $e; + } + } + + private function initializeJob() + : void + { + set_time_limit(self::MAX_EXECUTION_TIME); + $this->processedCount = 0; + $this->errorCount = 0; + } + + private function processPeriod() + : void + { + $disk = Storage::disk(self::DISK_NAME); + $filePath = "$this->period/" . self::FILENAME; + + if (!$this->validateFile($disk, $filePath)) { return; } - foreach ($this->periods as $period) { - // Skip the _parameter folder - /*if ($period === '_parameter') { - Log::info("Skipping _parameter folder"); - continue; - }*/ + $tempFilePath = $this->createTemporaryFile($disk, $filePath); + $this->processFile($tempFilePath, $filePath); + $this->cleanup($tempFilePath); + } - // Construct the filename based on the period folder name - $filename = "ST.STMT.NARR.FORMAT.csv"; - $filePath = "$period/$filename"; + private function validateFile($disk, string $filePath) + : bool + { + Log::info("Processing statement narrative format file: $filePath"); - Log::info("Processing statement narrative format file: $filePath"); - - if (!$disk->exists($filePath)) { - Log::warning("File not found: $filePath"); - continue; - } - - // Create a temporary local copy of the file - $tempFilePath = storage_path("app/temp_$filename"); - file_put_contents($tempFilePath, $disk->get($filePath)); - - $handle = fopen($tempFilePath, "r"); - - if ($handle !== false) { - $headers = (new TempStmtNarrFormat())->getFillable(); - $rowCount = 0; - - while (($row = fgetcsv($handle, 0, "~")) !== false) { - $rowCount++; - - if (count($headers) === count($row)) { - $data = array_combine($headers, $row); - try { - if (isset($data['_id']) && $data['_id'] !== 'id' && $data['_id'] !== '_id') { - TempStmtNarrFormat::updateOrCreate( - ['_id' => $data['_id']], - $data - ); - $processedCount++; - } - } catch (Exception $e) { - $errorCount++; - Log::error("Error processing Statement Narrative Format at row $rowCount in $filePath: " . $e->getMessage()); - } - } else { - Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row)); - } - } - - fclose($handle); - Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors."); - - // Clean up the temporary file - unlink($tempFilePath); - } else { - Log::error("Unable to open file: $filePath"); - } + if (!$disk->exists($filePath)) { + Log::warning("File not found: $filePath"); + return false; } - Log::info("Statement Narrative Format data processing completed. Total processed: $processedCount, Total errors: $errorCount"); + return true; + } - } catch (Exception $e) { - Log::error('Error in ProcessStmtNarrFormatDataJob: ' . $e->getMessage()); - throw $e; + private function createTemporaryFile($disk, string $filePath) + : string + { + $tempFilePath = storage_path("app/temp_" . self::FILENAME); + file_put_contents($tempFilePath, $disk->get($filePath)); + return $tempFilePath; + } + + private function processFile(string $tempFilePath, string $filePath) + : void + { + $handle = fopen($tempFilePath, "r"); + if ($handle === false) { + Log::error("Unable to open file: $filePath"); + return; + } + + $headers = (new TempStmtNarrFormat())->getFillable(); + $rowCount = 0; + + while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) { + $rowCount++; + $this->processRow($row, $headers, $rowCount, $filePath); + } + + fclose($handle); + Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors."); + } + + private function processRow(array $row, array $headers, int $rowCount, string $filePath) + : void + { + if (count($headers) !== count($row)) { + Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . + count($headers) . ", Got: " . count($row)); + return; + } + + $data = array_combine($headers, $row); + $this->saveRecord($data, $rowCount, $filePath); + } + + private function saveRecord(array $data, int $rowCount, string $filePath) + : void + { + try { + if (isset($data['_id']) && $data['_id'] !== 'id' && $data['_id'] !== '_id') { + TempStmtNarrFormat::updateOrCreate(['_id' => $data['_id']], $data); + $this->processedCount++; + } + } catch (Exception $e) { + $this->errorCount++; + Log::error("Error processing Statement Narrative Format at row $rowCount in $filePath: " . $e->getMessage()); + } + } + + private function cleanup(string $tempFilePath) + : void + { + if (file_exists($tempFilePath)) { + unlink($tempFilePath); + } + } + + private function logJobCompletion() + : void + { + Log::info("Statement Narrative Format data processing completed. " . + "Total processed: {$this->processedCount}, Total errors: {$this->errorCount}"); } } -} diff --git a/app/Jobs/ProcessStmtNarrParamDataJob.php b/app/Jobs/ProcessStmtNarrParamDataJob.php index 7b18db9..3990211 100644 --- a/app/Jobs/ProcessStmtNarrParamDataJob.php +++ b/app/Jobs/ProcessStmtNarrParamDataJob.php @@ -16,98 +16,149 @@ { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; - protected array $periods; + private const CSV_DELIMITER = '~'; + private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds + private const FILENAME = 'ST.STMT.NARR.PARAM.csv'; + private const DISK_NAME = 'sftpStatement'; + + private string $period; + private int $processedCount = 0; + private int $errorCount = 0; /** * Create a new job instance. */ - public function __construct(array|string $periods = []) + public function __construct(string $period = '') { - $this->periods = is_string($periods) ? [$periods] : $periods; + $this->period = $period; } - /** * Execute the job. */ - public function handle(): void + public function handle() + : void { try { - set_time_limit(24 * 60 * 60); - $disk = Storage::disk('sftpStatement'); - $processedCount = 0; - $errorCount = 0; + $this->initializeJob(); - if (empty($this->periods)) { + if ($this->period === '') { Log::warning('No periods provided for statement narrative parameter data processing'); return; } - foreach ($this->periods as $period) { - // Skip the _parameter folder - /*if ($period === '_parameter') { - Log::info("Skipping _parameter folder"); - continue; - }*/ - - // Construct the filename based on the period folder name - $filename = "ST.STMT.NARR.PARAM.csv"; - $filePath = "$period/$filename"; - - Log::info("Processing statement narrative parameter file: $filePath"); - - if (!$disk->exists($filePath)) { - Log::warning("File not found: $filePath"); - continue; - } - - // Create a temporary local copy of the file - $tempFilePath = storage_path("app/temp_$filename"); - file_put_contents($tempFilePath, $disk->get($filePath)); - - $handle = fopen($tempFilePath, "r"); - - if ($handle !== false) { - $headers = (new TempStmtNarrParam())->getFillable(); - $rowCount = 0; - - while (($row = fgetcsv($handle, 0, "~")) !== false) { - $rowCount++; - - if (count($headers) === count($row)) { - $data = array_combine($headers, $row); - try { - if (isset($data['_id']) && $data['_id'] !== 'id' && $data['_id'] !== '_id') { - TempStmtNarrParam::updateOrCreate( - ['_id' => $data['_id']], - $data - ); - $processedCount++; - } - } catch (Exception $e) { - $errorCount++; - Log::error("Error processing Statement Narrative Parameter at row $rowCount in $filePath: " . $e->getMessage()); - } - } else { - Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row)); - } - } - - fclose($handle); - Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors."); - - // Clean up the temporary file - unlink($tempFilePath); - } else { - Log::error("Unable to open file: $filePath"); - } - } - - Log::info("Statement Narrative Parameter data processing completed. Total processed: $processedCount, Total errors: $errorCount"); - + $this->processPeriod(); + $this->logJobCompletion(); } catch (Exception $e) { Log::error('Error in ProcessStmtNarrParamDataJob: ' . $e->getMessage()); throw $e; } } + + private function initializeJob() + : void + { + set_time_limit(self::MAX_EXECUTION_TIME); + $this->processedCount = 0; + $this->errorCount = 0; + } + + private function processPeriod() + : void + { + $disk = Storage::disk(self::DISK_NAME); + $filePath = "$this->period/" . self::FILENAME; + + if (!$this->validateFile($disk, $filePath)) { + return; + } + + $tempFilePath = $this->createTemporaryFile($disk, $filePath); + $this->processFile($tempFilePath, $filePath); + $this->cleanup($tempFilePath); + } + + private function validateFile($disk, string $filePath) + : bool + { + Log::info("Processing statement narrative parameter file: $filePath"); + + if (!$disk->exists($filePath)) { + Log::warning("File not found: $filePath"); + return false; + } + + return true; + } + + private function createTemporaryFile($disk, string $filePath) + : string + { + $tempFilePath = storage_path("app/temp_" . self::FILENAME); + file_put_contents($tempFilePath, $disk->get($filePath)); + return $tempFilePath; + } + + private function processFile(string $tempFilePath, string $filePath) + : void + { + $handle = fopen($tempFilePath, "r"); + if ($handle === false) { + Log::error("Unable to open file: $filePath"); + return; + } + + $headers = (new TempStmtNarrParam())->getFillable(); + $rowCount = 0; + + while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) { + $rowCount++; + $this->processRow($row, $headers, $rowCount, $filePath); + } + + fclose($handle); + Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors."); + } + + private function processRow(array $row, array $headers, int $rowCount, string $filePath) + : void + { + if (count($headers) !== count($row)) { + Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . + count($headers) . ", Got: " . count($row)); + return; + } + + $data = array_combine($headers, $row); + $this->saveRecord($data, $rowCount, $filePath); + } + + private function saveRecord(array $data, int $rowCount, string $filePath) + : void + { + try { + if (isset($data['_id']) && $data['_id'] !== 'id' && $data['_id'] !== '_id') { + TempStmtNarrParam::updateOrCreate(['_id' => $data['_id']], $data); + $this->processedCount++; + } + } catch (Exception $e) { + $this->errorCount++; + Log::error("Error processing Statement Narrative Parameter at row $rowCount in $filePath: " . $e->getMessage()); + } + } + + private function cleanup(string $tempFilePath) + : void + { + if (file_exists($tempFilePath)) { + unlink($tempFilePath); + } + } + + private function logJobCompletion() + : void + { + Log::info("Statement Narrative Parameter data processing completed. " . + "Total processed: {$this->processedCount}, Total errors: {$this->errorCount}"); + } } diff --git a/app/Jobs/ProcessTellerDataJob.php b/app/Jobs/ProcessTellerDataJob.php index 3aaddad..7a15e70 100644 --- a/app/Jobs/ProcessTellerDataJob.php +++ b/app/Jobs/ProcessTellerDataJob.php @@ -16,11 +16,11 @@ { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; - // Konstanta untuk nilai-nilai statis - private const FILE_EXTENSION = '.ST.TELLER.csv'; - private const CSV_DELIMITER = '~'; - private const DISK_NAME = 'sftpStatement'; - private const HEADER_MAP = [ + private const CSV_DELIMITER = '~'; + private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds + private const FILENAME = 'ST.TELLER.csv'; + private const DISK_NAME = 'sftpStatement'; + private const HEADER_MAP = [ 'id' => 'id_teller', 'account_1' => 'account_1', 'currency_1' => 'currency_1', @@ -126,8 +126,9 @@ 'last_version' => 'last_version' ]; - // Pemetaan bidang header ke kolom model - protected string $period; + private string $period; + private int $processedCount = 0; + private int $errorCount = 0; /** * Create a new job instance. @@ -144,120 +145,94 @@ : void { try { - set_time_limit(24 * 60 * 60); + $this->initializeJob(); if ($this->period === '') { Log::warning('No period provided for teller data processing'); return; } - $stats = $this->processPeriodFile(); - - Log::info("ProcessTellerDataJob completed. Total processed: {$stats['processed']}, Total errors: {$stats['errors']}"); + $this->processPeriod(); + $this->logJobCompletion(); } catch (Exception $e) { - Log::error("Error in ProcessTellerDataJob: " . $e->getMessage()); + Log::error('Error in ProcessTellerDataJob: ' . $e->getMessage()); throw $e; } } - /** - * Process a single period file - */ - private function processPeriodFile() - : array + private function initializeJob() + : void { - $disk = Storage::disk(self::DISK_NAME); - $filename = $this->period . self::FILE_EXTENSION; - $filePath = "{$this->period}/$filename"; - $processedCount = 0; - $errorCount = 0; + set_time_limit(self::MAX_EXECUTION_TIME); + $this->processedCount = 0; + $this->errorCount = 0; + } + private function processPeriod() + : void + { + $disk = Storage::disk(self::DISK_NAME); + $filename = "{$this->period}." . self::FILENAME; + $filePath = "{$this->period}/$filename"; + + if (!$this->validateFile($disk, $filePath)) { + return; + } + + $tempFilePath = $this->createTemporaryFile($disk, $filePath, $filename); + $this->processFile($tempFilePath, $filePath); + $this->cleanup($tempFilePath); + } + + private function validateFile($disk, string $filePath) + : bool + { Log::info("Processing teller file: $filePath"); if (!$disk->exists($filePath)) { Log::warning("File not found: $filePath"); - return ['processed' => 0, 'errors' => 0]; + return false; } - $tempFilePath = $this->createTempFile($disk, $filePath, $filename); - - $result = $this->processCSVFile($tempFilePath, $filePath); - $processedCount += $result['processed']; - $errorCount += $result['errors']; - - // Clean up the temporary file - if (file_exists($tempFilePath)) { - unlink($tempFilePath); - } - - Log::info("Completed processing $filePath. Processed {$result['processed']} records with {$result['errors']} errors."); - - return [ - 'processed' => $processedCount, - 'errors' => $errorCount - ]; + return true; } - /** - * Create a temporary file for processing - */ - private function createTempFile($disk, string $filePath, string $filename) + private function createTemporaryFile($disk, string $filePath, string $fileName) : string { - $tempFilePath = storage_path("app/temp_$filename"); + $tempFilePath = storage_path("app/temp_$fileName"); file_put_contents($tempFilePath, $disk->get($filePath)); return $tempFilePath; } - /** - * Process a CSV file and import data - */ - private function processCSVFile(string $tempFilePath, string $originalFilePath) - : array + private function processFile(string $tempFilePath, string $filePath) + : void { - $processedCount = 0; - $errorCount = 0; - $handle = fopen($tempFilePath, "r"); if ($handle === false) { - Log::error("Unable to open file: $originalFilePath"); - return ['processed' => 0, 'errors' => 0]; + Log::error("Unable to open file: $filePath"); + return; } // Get the headers from the first row $headerRow = fgetcsv($handle, 0, self::CSV_DELIMITER); if (!$headerRow) { fclose($handle); - return ['processed' => 0, 'errors' => 0]; + return; } $rowCount = 0; while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) { $rowCount++; - - if (count($headerRow) !== count($row)) { - Log::warning("Row $rowCount in $originalFilePath has incorrect column count. Expected: " . count($headerRow) . ", Got: " . count($row)); - continue; - } - - $result = $this->processRow($headerRow, $row, $rowCount, $originalFilePath); - $processedCount += $result['processed']; - $errorCount += $result['errors']; + $this->processRow($headerRow, $row, $rowCount, $filePath); } fclose($handle); - - return [ - 'processed' => $processedCount, - 'errors' => $errorCount - ]; + Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors."); } - /** - * Process a single row from the CSV file - */ private function processRow(array $headerRow, array $row, int $rowCount, string $filePath) - : array + : void { // Combine the header row with the data row $rawData = array_combine($headerRow, $row); @@ -270,7 +245,7 @@ // Skip header row if it was included in the data if ($data['id_teller'] === 'id') { - return ['processed' => 0, 'errors' => 0]; + return; } try { @@ -278,10 +253,24 @@ $teller->fill($data); $teller->save(); - return ['processed' => 1, 'errors' => 0]; + $this->processedCount++; } catch (Exception $e) { Log::error("Error processing Teller at row $rowCount in $filePath: " . $e->getMessage()); - return ['processed' => 0, 'errors' => 1]; + $this->errorCount++; + } + } + + private function logJobCompletion() + : void + { + Log::info("ProcessTellerDataJob completed. Total processed: {$this->processedCount}, Total errors: {$this->errorCount}"); + } + + private function cleanup(string $tempFilePath) + : void + { + if (file_exists($tempFilePath)) { + unlink($tempFilePath); } } } diff --git a/app/Jobs/ProcessTransactionDataJob.php b/app/Jobs/ProcessTransactionDataJob.php index 0064b53..1322ec4 100644 --- a/app/Jobs/ProcessTransactionDataJob.php +++ b/app/Jobs/ProcessTransactionDataJob.php @@ -1,113 +1,164 @@ periods = is_string($periods) ? [$periods] : $periods; - } + use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; + private const CSV_DELIMITER = '~'; + private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds + private const FILENAME = 'ST.TRANSACTION.csv'; + private const DISK_NAME = 'sftpStatement'; - /** - * Execute the job. - */ - public function handle(): void - { - try { - set_time_limit(24 * 60 * 60); - $disk = Storage::disk('sftpStatement'); - $processedCount = 0; - $errorCount = 0; + private string $period; + private int $processedCount = 0; + private int $errorCount = 0; - if (empty($this->periods)) { - Log::warning('No periods provided for transaction data processing'); + /** + * Create a new job instance. + */ + public function __construct(string $period = '') + { + $this->period = $period; + } + + /** + * Execute the job. + */ + public function handle() + : void + { + try { + $this->initializeJob(); + + if ($this->period === '') { + Log::warning('No periods provided for transaction data processing'); + return; + } + + $this->processPeriod(); + $this->logJobCompletion(); + } catch (Exception $e) { + Log::error('Error in ProcessTransactionDataJob: ' . $e->getMessage()); + throw $e; + } + } + + private function initializeJob() + : void + { + set_time_limit(self::MAX_EXECUTION_TIME); + $this->processedCount = 0; + $this->errorCount = 0; + } + + private function processPeriod() + : void + { + $disk = Storage::disk(self::DISK_NAME); + $filePath = "$this->period/" . self::FILENAME; + + if (!$this->validateFile($disk, $filePath)) { return; } - foreach ($this->periods as $period) { - // Skip the _parameter folder - /*if ($period === '_parameter') { - Log::info("Skipping _parameter folder"); - continue; - }*/ + $tempFilePath = $this->createTemporaryFile($disk, $filePath); + $this->processFile($tempFilePath, $filePath); + $this->cleanup($tempFilePath); + } - // Construct the filename based on the period folder name - $filename = "ST.TRANSACTION.csv"; - $filePath = "$period/$filename"; + private function validateFile($disk, string $filePath) + : bool + { + Log::info("Processing transaction file: $filePath"); - Log::info("Processing transaction file: $filePath"); - - if (!$disk->exists($filePath)) { - Log::warning("File not found: $filePath"); - continue; - } - - // Create a temporary local copy of the file - $tempFilePath = storage_path("app/temp_$filename"); - file_put_contents($tempFilePath, $disk->get($filePath)); - - $handle = fopen($tempFilePath, "r"); - - if ($handle !== false) { - $headers = (new TempTransaction())->getFillable(); - $rowCount = 0; - - while (($row = fgetcsv($handle, 0, "~")) !== false) { - $rowCount++; - - if (count($headers) === count($row)) { - $data = array_combine($headers, $row); - try { - if (isset($data['_id']) && $data['_id'] !== 'id' && $data['_id'] !== '_id') { - TempTransaction::updateOrCreate( - ['_id' => $data['_id']], - $data - ); - $processedCount++; - } - } catch (Exception $e) { - $errorCount++; - Log::error("Error processing Transaction at row $rowCount in $filePath: " . $e->getMessage()); - } - } else { - Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row)); - } - } - - fclose($handle); - Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors."); - - // Clean up the temporary file - unlink($tempFilePath); - } else { - Log::error("Unable to open file: $filePath"); - } + if (!$disk->exists($filePath)) { + Log::warning("File not found: $filePath"); + return false; } - Log::info("Transaction data processing completed. Total processed: $processedCount, Total errors: $errorCount"); + return true; + } - } catch (Exception $e) { - Log::error('Error in ProcessTransactionDataJob: ' . $e->getMessage()); - throw $e; + private function createTemporaryFile($disk, string $filePath) + : string + { + $tempFilePath = storage_path("app/temp_" . self::FILENAME); + file_put_contents($tempFilePath, $disk->get($filePath)); + return $tempFilePath; + } + + private function processFile(string $tempFilePath, string $filePath) + : void + { + $handle = fopen($tempFilePath, "r"); + if ($handle === false) { + Log::error("Unable to open file: $filePath"); + return; + } + + $headers = (new TempTransaction())->getFillable(); + $rowCount = 0; + + while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) { + $rowCount++; + $this->processRow($row, $headers, $rowCount, $filePath); + } + + fclose($handle); + Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors."); + } + + private function processRow(array $row, array $headers, int $rowCount, string $filePath) + : void + { + if (count($headers) !== count($row)) { + Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . + count($headers) . ", Got: " . count($row)); + return; + } + + $data = array_combine($headers, $row); + $this->saveRecord($data, $rowCount, $filePath); + } + + private function saveRecord(array $data, int $rowCount, string $filePath) + : void + { + try { + if (isset($data['_id']) && $data['_id'] !== 'id' && $data['_id'] !== '_id') { + TempTransaction::updateOrCreate(['_id' => $data['_id']], $data); + $this->processedCount++; + } + } catch (Exception $e) { + $this->errorCount++; + Log::error("Error processing Transaction at row $rowCount in $filePath: " . $e->getMessage()); + } + } + + private function cleanup(string $tempFilePath) + : void + { + if (file_exists($tempFilePath)) { + unlink($tempFilePath); + } + } + + private function logJobCompletion() + : void + { + Log::info("Transaction data processing completed. " . + "Total processed: {$this->processedCount}, Total errors: {$this->errorCount}"); } } -}