refactor(jobs): optimize job classes by adding modular methods and constants
- Menambahkan konstanta baru pada setiap job untuk meningkatkan keterbacaan: - `CSV_DELIMITER` untuk delimiter CSV. - `MAX_EXECUTION_TIME` untuk batas waktu eksekusi (86400 detik). - `FILENAME` untuk nama file masing-masing job. - `DISK_NAME` untuk disk yang digunakan. - Mengubah `protected` menjadi `private` untuk properti seperti: - `$period`, `$processedCount`, dan `$errorCount` di semua job. - Memindahkan logika proses menjadi metode modular untuk meningkatkan modularitas: - Metode `initializeJob` untuk inisialisasi counter. - Metode `processPeriod` untuk menangani file dan memulai proses. - Metode `validateFile` untuk validasi keberadaan file. - Metode `createTemporaryFile` untuk menyalin file sementara. - Metode `processFile` untuk membaca isi file CSV dan memproses. - Metode `processRow`/`mapAndSaveRecord`/`saveRecord` untuk pemrosesan dan penyimpanan data. - Metode `cleanup` untuk menghapus file sementara. - Metode `logJobCompletion` untuk logging hasil akhir. - Menerapkan pengolahan file dengan sistem logging terperinci: - Logging row dengan kolom tidak sesuai akan menghasilkan peringatan. - Mencatat jumlah record berhasil diproses serta error. Refaktor ini bertujuan untuk meningkatkan kualitas kode melalui modularisasi, keterbacaan, dan kemudahan pengujian ulang di seluruh kelas job.
This commit is contained in:
daengdeni
@@ -17,7 +17,13 @@
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
protected $period;
|
||||
private const CSV_DELIMITER = '~';
|
||||
private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds
|
||||
private const FILENAME = 'ST.ACCOUNT.csv';
|
||||
private const DISK_NAME = 'sftpStatement';
|
||||
private string $period;
|
||||
private int $processedCount = 0;
|
||||
private int $errorCount = 0;
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
@@ -34,107 +40,172 @@
|
||||
: void
|
||||
{
|
||||
try {
|
||||
set_time_limit(24 * 60 * 60);
|
||||
$disk = Storage::disk('sftpStatement');
|
||||
$processedCount = 0;
|
||||
$errorCount = 0;
|
||||
$this->initializeJob();
|
||||
|
||||
if ($this->period === '') {
|
||||
Log::warning('No period provided for account data processing');
|
||||
return;
|
||||
}
|
||||
|
||||
// Construct the filename based on the period folder name
|
||||
$filename = "{$this->period}.ST.ACCOUNT.csv";
|
||||
$filePath = "{$this->period}/$filename";
|
||||
|
||||
Log::info("Processing account file: $filePath");
|
||||
|
||||
if (!$disk->exists($filePath)) {
|
||||
Log::warning("File not found: $filePath");
|
||||
return;
|
||||
}
|
||||
|
||||
// Create a temporary local copy of the file
|
||||
$tempFilePath = storage_path("app/temp_$filename");
|
||||
file_put_contents($tempFilePath, $disk->get($filePath));
|
||||
|
||||
$handle = fopen($tempFilePath, "r");
|
||||
|
||||
if ($handle !== false) {
|
||||
$headers = (new Account())->getFillable();
|
||||
Log::info('Headers: ' . implode(", ", $headers));
|
||||
$rowCount = 0;
|
||||
|
||||
while (($row = fgetcsv($handle, 0, "~")) !== false) {
|
||||
$rowCount++;
|
||||
|
||||
if (count($headers) === count($row)) {
|
||||
$data = array_combine($headers, $row);
|
||||
|
||||
// Check if start_year_bal is empty and set it to 0 if so
|
||||
if (empty($data['start_year_bal']) || $data['start_year_bal'] == "" || $data['start_year_bal'] == null) {
|
||||
$data['start_year_bal'] = 0;
|
||||
}
|
||||
|
||||
if (empty($data['closure_date']) || $data['closure_date'] == "" || $data['closure_date'] == null) {
|
||||
$data['closure_date'] = null;
|
||||
}
|
||||
|
||||
try {
|
||||
if ($data['account_number'] !== 'account_number') {
|
||||
// Use firstOrNew instead of updateOrCreate
|
||||
$account = Account::firstOrNew(['account_number' => $data['account_number']]);
|
||||
$account->fill($data);
|
||||
$account->save();
|
||||
|
||||
// Store the opening balances in the AccountBalance model for this period
|
||||
if (isset($data['open_actual_bal']) || isset($data['open_cleared_bal'])) {
|
||||
// Prepare balance data for bulk insert/update
|
||||
$balanceData = [
|
||||
'account_number' => $data['account_number'],
|
||||
'period' => $this->period,
|
||||
'actual_balance' => empty($data['open_actual_bal']) ? 0 : $data['open_actual_bal'],
|
||||
'cleared_balance' => empty($data['open_cleared_bal']) ? 0 : $data['open_cleared_bal'],
|
||||
'created_at' => now(),
|
||||
'updated_at' => now()
|
||||
];
|
||||
|
||||
// Use updateOrInsert to reduce queries
|
||||
AccountBalance::updateOrInsert(
|
||||
[
|
||||
'account_number' => $data['account_number'],
|
||||
'period' => $this->period
|
||||
],
|
||||
$balanceData
|
||||
);
|
||||
}
|
||||
|
||||
$processedCount++;
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
$errorCount++;
|
||||
Log::error("Error processing Account at row $rowCount in $filePath: " . $e->getMessage());
|
||||
}
|
||||
} else {
|
||||
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row));
|
||||
}
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors.");
|
||||
|
||||
// Clean up the temporary file
|
||||
unlink($tempFilePath);
|
||||
} else {
|
||||
Log::error("Unable to open file: $filePath");
|
||||
}
|
||||
|
||||
Log::info("Account data processing completed. Total processed: $processedCount, Total errors: $errorCount");
|
||||
|
||||
$this->processPeriod();
|
||||
$this->logJobCompletion();
|
||||
} catch (Exception $e) {
|
||||
Log::error('Error in ProcessAccountDataJob: ' . $e->getMessage());
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
private function initializeJob()
|
||||
: void
|
||||
{
|
||||
set_time_limit(self::MAX_EXECUTION_TIME);
|
||||
$this->processedCount = 0;
|
||||
$this->errorCount = 0;
|
||||
}
|
||||
|
||||
private function processPeriod()
|
||||
: void
|
||||
{
|
||||
$disk = Storage::disk(self::DISK_NAME);
|
||||
$filename = "{$this->period}." . self::FILENAME;
|
||||
$filePath = "{$this->period}/$filename";
|
||||
|
||||
if (!$this->validateFile($disk, $filePath)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$tempFilePath = $this->createTemporaryFile($disk, $filePath, $filename);
|
||||
$this->processFile($tempFilePath, $filePath);
|
||||
$this->cleanup($tempFilePath);
|
||||
}
|
||||
|
||||
private function validateFile($disk, string $filePath)
|
||||
: bool
|
||||
{
|
||||
Log::info("Processing account file: $filePath");
|
||||
|
||||
if (!$disk->exists($filePath)) {
|
||||
Log::warning("File not found: $filePath");
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private function createTemporaryFile($disk, string $filePath, string $filename)
|
||||
: string
|
||||
{
|
||||
$tempFilePath = storage_path("app/temp_$filename");
|
||||
file_put_contents($tempFilePath, $disk->get($filePath));
|
||||
return $tempFilePath;
|
||||
}
|
||||
|
||||
private function processFile(string $tempFilePath, string $filePath)
|
||||
: void
|
||||
{
|
||||
$handle = fopen($tempFilePath, "r");
|
||||
if ($handle === false) {
|
||||
Log::error("Unable to open file: $filePath");
|
||||
return;
|
||||
}
|
||||
|
||||
$headers = (new Account())->getFillable();
|
||||
Log::info('Headers: ' . implode(", ", $headers));
|
||||
$rowCount = 0;
|
||||
|
||||
while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) {
|
||||
$rowCount++;
|
||||
$this->processRow($row, $headers, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors.");
|
||||
}
|
||||
|
||||
private function processRow(array $row, array $headers, int $rowCount, string $filePath)
|
||||
: void
|
||||
{
|
||||
if (count($headers) !== count($row)) {
|
||||
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " .
|
||||
count($headers) . ", Got: " . count($row));
|
||||
return;
|
||||
}
|
||||
|
||||
$data = array_combine($headers, $row);
|
||||
$this->normalizeData($data);
|
||||
$this->saveRecord($data, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
private function normalizeData(array &$data)
|
||||
: void
|
||||
{
|
||||
// Check if start_year_bal is empty and set it to 0 if so
|
||||
if (empty($data['start_year_bal']) || $data['start_year_bal'] == "" || $data['start_year_bal'] == null) {
|
||||
$data['start_year_bal'] = 0;
|
||||
}
|
||||
|
||||
if (empty($data['closure_date']) || $data['closure_date'] == "" || $data['closure_date'] == null) {
|
||||
$data['closure_date'] = null;
|
||||
}
|
||||
}
|
||||
|
||||
private function saveRecord(array $data, int $rowCount, string $filePath)
|
||||
: void
|
||||
{
|
||||
try {
|
||||
if ($data['account_number'] !== 'account_number') {
|
||||
// Use firstOrNew instead of updateOrCreate
|
||||
$account = Account::firstOrNew(['account_number' => $data['account_number']]);
|
||||
$account->fill($data);
|
||||
$account->save();
|
||||
|
||||
$this->saveAccountBalance($data);
|
||||
$this->processedCount++;
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
$this->errorCount++;
|
||||
Log::error("Error processing Account at row $rowCount in $filePath: " . $e->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private function saveAccountBalance(array $data)
|
||||
: void
|
||||
{
|
||||
// Store the opening balances in the AccountBalance model for this period
|
||||
if (isset($data['open_actual_bal']) || isset($data['open_cleared_bal'])) {
|
||||
// Prepare balance data for bulk insert/update
|
||||
$balanceData = [
|
||||
'account_number' => $data['account_number'],
|
||||
'period' => $this->period,
|
||||
'actual_balance' => empty($data['open_actual_bal']) ? 0 : $data['open_actual_bal'],
|
||||
'cleared_balance' => empty($data['open_cleared_bal']) ? 0 : $data['open_cleared_bal'],
|
||||
'created_at' => now(),
|
||||
'updated_at' => now()
|
||||
];
|
||||
|
||||
// Use updateOrInsert to reduce queries
|
||||
AccountBalance::updateOrInsert(
|
||||
[
|
||||
'account_number' => $data['account_number'],
|
||||
'period' => $this->period
|
||||
],
|
||||
$balanceData
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private function cleanup(string $tempFilePath)
|
||||
: void
|
||||
{
|
||||
if (file_exists($tempFilePath)) {
|
||||
unlink($tempFilePath);
|
||||
}
|
||||
}
|
||||
|
||||
private function logJobCompletion()
|
||||
: void
|
||||
{
|
||||
Log::info("Account data processing completed. " .
|
||||
"Total processed: {$this->processedCount}, Total errors: {$this->errorCount}");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,7 +16,14 @@
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
protected string $period;
|
||||
private const CSV_DELIMITER = '~';
|
||||
private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds
|
||||
private const FILENAME = 'ST.AA.ARRANGEMENT.csv';
|
||||
private const DISK_NAME = 'sftpStatement';
|
||||
|
||||
private string $period;
|
||||
private int $processedCount = 0;
|
||||
private int $errorCount = 0;
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
@@ -29,76 +36,133 @@
|
||||
/**
|
||||
* Execute the job.
|
||||
*/
|
||||
public function handle(): void
|
||||
public function handle()
|
||||
: void
|
||||
{
|
||||
try {
|
||||
set_time_limit(24 * 60 * 60);
|
||||
$disk = Storage::disk('sftpStatement');
|
||||
$processedCount = 0;
|
||||
$errorCount = 0;
|
||||
$this->initializeJob();
|
||||
|
||||
if ($this->period === '') {
|
||||
Log::warning('No period provided for arrangement data processing');
|
||||
return;
|
||||
}
|
||||
|
||||
// Construct the filename based on the period folder name
|
||||
$filename = "{$this->period}.ST.AA.ARRANGEMENT.csv";
|
||||
$filePath = "{$this->period}/$filename";
|
||||
|
||||
Log::info("Processing arrangement file: $filePath");
|
||||
|
||||
if (!$disk->exists($filePath)) {
|
||||
Log::warning("File not found: $filePath");
|
||||
return;
|
||||
}
|
||||
|
||||
// Create a temporary local copy of the file
|
||||
$tempFilePath = storage_path("app/temp_$filename");
|
||||
file_put_contents($tempFilePath, $disk->get($filePath));
|
||||
|
||||
$handle = fopen($tempFilePath, "r");
|
||||
|
||||
if ($handle !== false) {
|
||||
$headers = (new TempArrangement())->getFillable();
|
||||
$rowCount = 0;
|
||||
|
||||
while (($row = fgetcsv($handle, 0, "~")) !== false) {
|
||||
$rowCount++;
|
||||
|
||||
if (count($headers) === count($row)) {
|
||||
$data = array_combine($headers, $row);
|
||||
try {
|
||||
if ($data['arrangement_id'] !== 'arrangement_id') {
|
||||
TempArrangement::updateOrCreate(
|
||||
['arrangement_id' => $data['arrangement_id']], // key to find existing record
|
||||
$data // data to update or create
|
||||
);
|
||||
$processedCount++;
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
$errorCount++;
|
||||
Log::error("Error processing Arrangement at row $rowCount in $filePath: " . $e->getMessage());
|
||||
}
|
||||
} else {
|
||||
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row));
|
||||
}
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors.");
|
||||
|
||||
// Clean up the temporary file
|
||||
unlink($tempFilePath);
|
||||
} else {
|
||||
Log::error("Unable to open file: $filePath");
|
||||
}
|
||||
|
||||
Log::info("Arrangement data processing completed. Total processed: $processedCount, Total errors: $errorCount");
|
||||
|
||||
$this->processPeriod();
|
||||
$this->logJobCompletion();
|
||||
} catch (Exception $e) {
|
||||
Log::error('Error in ProcessArrangementDataJob: ' . $e->getMessage());
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
private function initializeJob()
|
||||
: void
|
||||
{
|
||||
set_time_limit(self::MAX_EXECUTION_TIME);
|
||||
$this->processedCount = 0;
|
||||
$this->errorCount = 0;
|
||||
}
|
||||
|
||||
private function processPeriod()
|
||||
: void
|
||||
{
|
||||
$disk = Storage::disk(self::DISK_NAME);
|
||||
$filename = "{$this->period}." . self::FILENAME;
|
||||
$filePath = "{$this->period}/$filename";
|
||||
|
||||
if (!$this->validateFile($disk, $filePath)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$tempFilePath = $this->createTemporaryFile($disk, $filePath, $filename);
|
||||
$this->processFile($tempFilePath, $filePath);
|
||||
$this->cleanup($tempFilePath);
|
||||
}
|
||||
|
||||
private function validateFile($disk, string $filePath)
|
||||
: bool
|
||||
{
|
||||
Log::info("Processing arrangement file: $filePath");
|
||||
|
||||
if (!$disk->exists($filePath)) {
|
||||
Log::warning("File not found: $filePath");
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private function createTemporaryFile($disk, string $filePath, string $filename)
|
||||
: string
|
||||
{
|
||||
$tempFilePath = storage_path("app/temp_$filename");
|
||||
file_put_contents($tempFilePath, $disk->get($filePath));
|
||||
return $tempFilePath;
|
||||
}
|
||||
|
||||
private function processFile(string $tempFilePath, string $filePath)
|
||||
: void
|
||||
{
|
||||
$handle = fopen($tempFilePath, "r");
|
||||
if ($handle === false) {
|
||||
Log::error("Unable to open file: $filePath");
|
||||
return;
|
||||
}
|
||||
|
||||
$headers = (new TempArrangement())->getFillable();
|
||||
$rowCount = 0;
|
||||
|
||||
while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) {
|
||||
$rowCount++;
|
||||
$this->processRow($row, $headers, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors.");
|
||||
}
|
||||
|
||||
private function processRow(array $row, array $headers, int $rowCount, string $filePath)
|
||||
: void
|
||||
{
|
||||
if (count($headers) !== count($row)) {
|
||||
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " .
|
||||
count($headers) . ", Got: " . count($row));
|
||||
return;
|
||||
}
|
||||
|
||||
$data = array_combine($headers, $row);
|
||||
$this->saveRecord($data, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
private function saveRecord(array $data, int $rowCount, string $filePath)
|
||||
: void
|
||||
{
|
||||
try {
|
||||
if ($data['arrangement_id'] !== 'arrangement_id') {
|
||||
TempArrangement::updateOrCreate(
|
||||
['arrangement_id' => $data['arrangement_id']], // key to find existing record
|
||||
$data // data to update or create
|
||||
);
|
||||
$this->processedCount++;
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
$this->errorCount++;
|
||||
Log::error("Error processing Arrangement at row $rowCount in $filePath: " . $e->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private function cleanup(string $tempFilePath)
|
||||
: void
|
||||
{
|
||||
if (file_exists($tempFilePath)) {
|
||||
unlink($tempFilePath);
|
||||
}
|
||||
}
|
||||
|
||||
private function logJobCompletion()
|
||||
: void
|
||||
{
|
||||
Log::info("Arrangement data processing completed. " .
|
||||
"Total processed: {$this->processedCount}, Total errors: {$this->errorCount}");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,11 +16,11 @@
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
// Konstanta untuk nilai-nilai statis
|
||||
private const FILE_EXTENSION = '.ST.ATM.TRANSACTION.csv';
|
||||
private const CSV_DELIMITER = '~';
|
||||
private const DISK_NAME = 'sftpStatement';
|
||||
private const HEADER_MAP = [
|
||||
private const CSV_DELIMITER = '~';
|
||||
private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds
|
||||
private const FILENAME = 'ST.ATM.TRANSACTION.csv';
|
||||
private const DISK_NAME = 'sftpStatement';
|
||||
private const HEADER_MAP = [
|
||||
'id' => 'transaction_id',
|
||||
'card_acc_id' => 'card_acc_id',
|
||||
'pan_number' => 'pan_number',
|
||||
@@ -40,8 +40,9 @@
|
||||
'proc_code' => 'proc_code'
|
||||
];
|
||||
|
||||
// Pemetaan bidang header ke kolom model
|
||||
protected string $period;
|
||||
private string $period;
|
||||
private int $processedCount = 0;
|
||||
private int $errorCount = 0;
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
@@ -58,64 +59,59 @@
|
||||
: void
|
||||
{
|
||||
try {
|
||||
set_time_limit(24 * 60 * 60);
|
||||
$this->initializeJob();
|
||||
|
||||
if ($this->period === '') {
|
||||
Log::warning('No period provided for ATM transaction data processing');
|
||||
return;
|
||||
}
|
||||
|
||||
$stats = $this->processPeriodFile();
|
||||
|
||||
Log::info("ProcessAtmTransactionJob completed. Total processed: {$stats['processed']}, Total errors: {$stats['errors']}");
|
||||
$this->processPeriod();
|
||||
$this->logJobCompletion();
|
||||
} catch (Exception $e) {
|
||||
Log::error("Error in ProcessAtmTransactionJob: " . $e->getMessage());
|
||||
Log::error('Error in ProcessAtmTransactionJob: ' . $e->getMessage());
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a single period file
|
||||
*/
|
||||
private function processPeriodFile()
|
||||
: array
|
||||
private function initializeJob()
|
||||
: void
|
||||
{
|
||||
$disk = Storage::disk(self::DISK_NAME);
|
||||
$filename = $this->period . self::FILE_EXTENSION;
|
||||
$filePath = "{$this->period}/$filename";
|
||||
$processedCount = 0;
|
||||
$errorCount = 0;
|
||||
set_time_limit(self::MAX_EXECUTION_TIME);
|
||||
$this->processedCount = 0;
|
||||
$this->errorCount = 0;
|
||||
}
|
||||
|
||||
private function processPeriod()
|
||||
: void
|
||||
{
|
||||
$disk = Storage::disk(self::DISK_NAME);
|
||||
$filename = "{$this->period}." . self::FILENAME;
|
||||
$filePath = "{$this->period}/$filename";
|
||||
|
||||
if (!$this->validateFile($disk, $filePath)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$tempFilePath = $this->createTemporaryFile($disk, $filePath, $filename);
|
||||
$this->processFile($tempFilePath, $filePath);
|
||||
$this->cleanup($tempFilePath);
|
||||
}
|
||||
|
||||
private function validateFile($disk, string $filePath)
|
||||
: bool
|
||||
{
|
||||
Log::info("Processing ATM transaction file: $filePath");
|
||||
|
||||
if (!$disk->exists($filePath)) {
|
||||
Log::warning("File not found: $filePath");
|
||||
return ['processed' => 0, 'errors' => 0];
|
||||
return false;
|
||||
}
|
||||
|
||||
$tempFilePath = $this->createTempFile($disk, $filePath, $filename);
|
||||
|
||||
$result = $this->processCSVFile($tempFilePath, $filePath);
|
||||
$processedCount += $result['processed'];
|
||||
$errorCount += $result['errors'];
|
||||
|
||||
// Clean up the temporary file
|
||||
if (file_exists($tempFilePath)) {
|
||||
unlink($tempFilePath);
|
||||
}
|
||||
|
||||
Log::info("Completed processing $filePath. Processed {$result['processed']} records with {$result['errors']} errors.");
|
||||
|
||||
return [
|
||||
'processed' => $processedCount,
|
||||
'errors' => $errorCount
|
||||
];
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a temporary file for processing
|
||||
*/
|
||||
private function createTempFile($disk, string $filePath, string $filename)
|
||||
private function createTemporaryFile($disk, string $filePath, string $filename)
|
||||
: string
|
||||
{
|
||||
$tempFilePath = storage_path("app/temp_$filename");
|
||||
@@ -123,59 +119,49 @@
|
||||
return $tempFilePath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a CSV file and import data
|
||||
*/
|
||||
private function processCSVFile(string $tempFilePath, string $originalFilePath)
|
||||
: array
|
||||
private function processFile(string $tempFilePath, string $filePath)
|
||||
: void
|
||||
{
|
||||
$processedCount = 0;
|
||||
$errorCount = 0;
|
||||
|
||||
$handle = fopen($tempFilePath, "r");
|
||||
if ($handle === false) {
|
||||
Log::error("Unable to open file: $originalFilePath");
|
||||
return ['processed' => 0, 'errors' => 0];
|
||||
Log::error("Unable to open file: $filePath");
|
||||
return;
|
||||
}
|
||||
|
||||
// Get the headers from the first row
|
||||
$headerRow = fgetcsv($handle, 0, self::CSV_DELIMITER);
|
||||
if (!$headerRow) {
|
||||
fclose($handle);
|
||||
return ['processed' => 0, 'errors' => 0];
|
||||
return;
|
||||
}
|
||||
|
||||
$rowCount = 0;
|
||||
while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) {
|
||||
$rowCount++;
|
||||
|
||||
if (count($headerRow) !== count($row)) {
|
||||
Log::warning("Row $rowCount in $originalFilePath has incorrect column count. Expected: " . count($headerRow) . ", Got: " . count($row));
|
||||
continue;
|
||||
}
|
||||
|
||||
$result = $this->processRow($headerRow, $row, $rowCount, $originalFilePath);
|
||||
$processedCount += $result['processed'];
|
||||
$errorCount += $result['errors'];
|
||||
$this->processRow($headerRow, $row, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
|
||||
return [
|
||||
'processed' => $processedCount,
|
||||
'errors' => $errorCount
|
||||
];
|
||||
Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors.");
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a single row from the CSV file
|
||||
*/
|
||||
private function processRow(array $headerRow, array $row, int $rowCount, string $filePath)
|
||||
: array
|
||||
: void
|
||||
{
|
||||
if (count($headerRow) !== count($row)) {
|
||||
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " .
|
||||
count($headerRow) . ", Got: " . count($row));
|
||||
return;
|
||||
}
|
||||
|
||||
// Combine the header row with the data row
|
||||
$rawData = array_combine($headerRow, $row);
|
||||
$this->mapAndSaveRecord($rawData, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
private function mapAndSaveRecord(array $rawData, int $rowCount, string $filePath)
|
||||
: void
|
||||
{
|
||||
// Map the raw data to our model fields
|
||||
$data = [];
|
||||
foreach (self::HEADER_MAP as $csvField => $modelField) {
|
||||
@@ -184,18 +170,16 @@
|
||||
|
||||
// Skip header row if it was included in the data
|
||||
if ($data['transaction_id'] === 'id') {
|
||||
return ['processed' => 0, 'errors' => 0];
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// Format dates if needed
|
||||
/*if (!empty($data['booking_date'])) {
|
||||
$data['booking_date'] = date('Y-m-d H:i:s', strtotime($data['booking_date']));
|
||||
}
|
||||
$this->saveRecord($data, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
if (!empty($data['value_date'])) {
|
||||
$data['value_date'] = date('Y-m-d H:i:s', strtotime($data['value_date']));
|
||||
}*/
|
||||
private function saveRecord(array $data, int $rowCount, string $filePath)
|
||||
: void
|
||||
{
|
||||
try {
|
||||
|
||||
// Create or update the record
|
||||
AtmTransaction::updateOrCreate(
|
||||
@@ -203,10 +187,25 @@
|
||||
$data
|
||||
);
|
||||
|
||||
return ['processed' => 1, 'errors' => 0];
|
||||
$this->processedCount++;
|
||||
} catch (Exception $e) {
|
||||
$this->errorCount++;
|
||||
Log::error("Error processing row $rowCount in $filePath: " . $e->getMessage());
|
||||
return ['processed' => 0, 'errors' => 1];
|
||||
}
|
||||
}
|
||||
|
||||
private function cleanup(string $tempFilePath)
|
||||
: void
|
||||
{
|
||||
if (file_exists($tempFilePath)) {
|
||||
unlink($tempFilePath);
|
||||
}
|
||||
}
|
||||
|
||||
private function logJobCompletion()
|
||||
: void
|
||||
{
|
||||
Log::info("ATM transaction data processing completed. " .
|
||||
"Total processed: {$this->processedCount}, Total errors: {$this->errorCount}");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,7 +16,14 @@
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
protected string $period;
|
||||
private const CSV_DELIMITER = '~';
|
||||
private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds
|
||||
private const FILENAME = 'ST.AA.BILL.DETAILS.csv';
|
||||
private const DISK_NAME = 'sftpStatement';
|
||||
|
||||
private string $period;
|
||||
private int $processedCount = 0;
|
||||
private int $errorCount = 0;
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
@@ -30,76 +37,132 @@
|
||||
* Execute the job.
|
||||
*/
|
||||
public function handle()
|
||||
: void
|
||||
{
|
||||
try {
|
||||
set_time_limit(24 * 60 * 60);
|
||||
$disk = Storage::disk('sftpStatement');
|
||||
$processedCount = 0;
|
||||
$errorCount = 0;
|
||||
$this->initializeJob();
|
||||
|
||||
if ($this->period === '') {
|
||||
Log::warning('No period provided for bill detail data processing');
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
// Construct the filename based on the period folder name
|
||||
$filename = "{$this->period}.ST.AA.BILL.DETAILS.csv";
|
||||
$filePath = "{$this->period}/$filename";
|
||||
|
||||
Log::info("Processing bill detail file: $filePath");
|
||||
|
||||
if (!$disk->exists($filePath)) {
|
||||
Log::warning("File not found: $filePath");
|
||||
return;
|
||||
}
|
||||
|
||||
// Create a temporary local copy of the file
|
||||
$tempFilePath = storage_path("app/temp_$filename");
|
||||
file_put_contents($tempFilePath, $disk->get($filePath));
|
||||
|
||||
$handle = fopen($tempFilePath, "r");
|
||||
|
||||
if ($handle !== false) {
|
||||
$headers = (new TempBillDetail())->getFillable();
|
||||
$rowCount = 0;
|
||||
|
||||
while (($row = fgetcsv($handle, 0, "~")) !== false) {
|
||||
$rowCount++;
|
||||
|
||||
if (count($headers) === count($row)) {
|
||||
$data = array_combine($headers, $row);
|
||||
try {
|
||||
if (isset($data['_id']) && $data['_id'] !== '_id') {
|
||||
TempBillDetail::updateOrCreate(
|
||||
['_id' => $data['_id']],
|
||||
$data
|
||||
);
|
||||
$processedCount++;
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
$errorCount++;
|
||||
Log::error("Error processing Bill Detail at row $rowCount in $filePath: " . $e->getMessage());
|
||||
}
|
||||
} else {
|
||||
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row));
|
||||
}
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors.");
|
||||
|
||||
// Clean up the temporary file
|
||||
unlink($tempFilePath);
|
||||
} else {
|
||||
Log::error("Unable to open file: $filePath");
|
||||
}
|
||||
|
||||
Log::info("Bill Detail data processing completed. Total processed: $processedCount, Total errors: $errorCount");
|
||||
|
||||
$this->processPeriod();
|
||||
$this->logJobCompletion();
|
||||
} catch (Exception $e) {
|
||||
Log::error('Error in ProcessBillDetailDataJob: ' . $e->getMessage());
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
private function initializeJob()
|
||||
: void
|
||||
{
|
||||
set_time_limit(self::MAX_EXECUTION_TIME);
|
||||
$this->processedCount = 0;
|
||||
$this->errorCount = 0;
|
||||
}
|
||||
|
||||
private function processPeriod()
|
||||
: void
|
||||
{
|
||||
$disk = Storage::disk(self::DISK_NAME);
|
||||
$filename = "{$this->period}." . self::FILENAME;
|
||||
$filePath = "{$this->period}/$filename";
|
||||
|
||||
if (!$this->validateFile($disk, $filePath)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$tempFilePath = $this->createTemporaryFile($disk, $filePath, $filename);
|
||||
$this->processFile($tempFilePath, $filePath);
|
||||
$this->cleanup($tempFilePath);
|
||||
}
|
||||
|
||||
private function validateFile($disk, string $filePath)
|
||||
: bool
|
||||
{
|
||||
Log::info("Processing bill detail file: $filePath");
|
||||
|
||||
if (!$disk->exists($filePath)) {
|
||||
Log::warning("File not found: $filePath");
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private function createTemporaryFile($disk, string $filePath, string $filename)
|
||||
: string
|
||||
{
|
||||
$tempFilePath = storage_path("app/temp_$filename");
|
||||
file_put_contents($tempFilePath, $disk->get($filePath));
|
||||
return $tempFilePath;
|
||||
}
|
||||
|
||||
private function processFile(string $tempFilePath, string $filePath)
|
||||
: void
|
||||
{
|
||||
$handle = fopen($tempFilePath, "r");
|
||||
if ($handle === false) {
|
||||
Log::error("Unable to open file: $filePath");
|
||||
return;
|
||||
}
|
||||
|
||||
$headers = (new TempBillDetail())->getFillable();
|
||||
$rowCount = 0;
|
||||
|
||||
while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) {
|
||||
$rowCount++;
|
||||
$this->processRow($row, $headers, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors.");
|
||||
}
|
||||
|
||||
private function processRow(array $row, array $headers, int $rowCount, string $filePath)
|
||||
: void
|
||||
{
|
||||
if (count($headers) !== count($row)) {
|
||||
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " .
|
||||
count($headers) . ", Got: " . count($row));
|
||||
return;
|
||||
}
|
||||
|
||||
$data = array_combine($headers, $row);
|
||||
$this->saveRecord($data, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
private function saveRecord(array $data, int $rowCount, string $filePath)
|
||||
: void
|
||||
{
|
||||
try {
|
||||
if (isset($data['_id']) && $data['_id'] !== '_id') {
|
||||
TempBillDetail::updateOrCreate(
|
||||
['_id' => $data['_id']],
|
||||
$data
|
||||
);
|
||||
$this->processedCount++;
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
$this->errorCount++;
|
||||
Log::error("Error processing Bill Detail at row $rowCount in $filePath: " . $e->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private function cleanup(string $tempFilePath)
|
||||
: void
|
||||
{
|
||||
if (file_exists($tempFilePath)) {
|
||||
unlink($tempFilePath);
|
||||
}
|
||||
}
|
||||
|
||||
private function logJobCompletion()
|
||||
: void
|
||||
{
|
||||
Log::info("Bill Detail data processing completed. " .
|
||||
"Total processed: {$this->processedCount}, Total errors: {$this->errorCount}");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,7 +16,26 @@
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
protected $period;
|
||||
private const CSV_DELIMITER = '~';
|
||||
private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds
|
||||
private const FILENAME = 'ST.CATEGORY.csv';
|
||||
private const DISK_NAME = 'sftpStatement';
|
||||
private const HEADER_MAP = [
|
||||
'id' => 'id_category',
|
||||
'date_time' => 'date_time',
|
||||
'description' => 'description',
|
||||
'short_name' => 'short_name',
|
||||
'system_ind' => 'system_ind',
|
||||
'record_status' => 'record_status',
|
||||
'co_code' => 'co_code',
|
||||
'curr_no' => 'curr_no',
|
||||
'l_db_cr_ind' => 'l_db_cr_ind',
|
||||
'category_code' => 'category_code'
|
||||
];
|
||||
|
||||
private string $period;
|
||||
private int $processedCount = 0;
|
||||
private int $errorCount = 0;
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
@@ -33,98 +52,151 @@
|
||||
: void
|
||||
{
|
||||
try {
|
||||
set_time_limit(24 * 60 * 60);
|
||||
$disk = Storage::disk('sftpStatement');
|
||||
$processedCount = 0;
|
||||
$errorCount = 0;
|
||||
$this->initializeJob();
|
||||
|
||||
if ($this->period === '') {
|
||||
Log::warning('No period provided for category data processing');
|
||||
return;
|
||||
}
|
||||
|
||||
// Construct the filename based on the period folder name
|
||||
$filename = "{$this->period}.ST.CATEGORY.csv";
|
||||
$filePath = "{$this->period}/$filename";
|
||||
|
||||
Log::info("Processing category file: $filePath");
|
||||
|
||||
if (!$disk->exists($filePath)) {
|
||||
Log::warning("File not found: $filePath");
|
||||
return;
|
||||
}
|
||||
|
||||
// Create a temporary local copy of the file
|
||||
$tempFilePath = storage_path("app/temp_$filename");
|
||||
file_put_contents($tempFilePath, $disk->get($filePath));
|
||||
|
||||
$handle = fopen($tempFilePath, "r");
|
||||
|
||||
if ($handle !== false) {
|
||||
// Get the headers from the first row
|
||||
$headerRow = fgetcsv($handle, 0, "~");
|
||||
|
||||
// Map the headers to our model fields
|
||||
$headerMap = [
|
||||
'id' => 'id_category',
|
||||
'date_time' => 'date_time',
|
||||
'description' => 'description',
|
||||
'short_name' => 'short_name',
|
||||
'system_ind' => 'system_ind',
|
||||
'record_status' => 'record_status',
|
||||
'co_code' => 'co_code',
|
||||
'curr_no' => 'curr_no',
|
||||
'l_db_cr_ind' => 'l_db_cr_ind',
|
||||
'category_code' => 'category_code'
|
||||
];
|
||||
|
||||
$rowCount = 0;
|
||||
|
||||
while (($row = fgetcsv($handle, 0, "~")) !== false) {
|
||||
$rowCount++;
|
||||
|
||||
if (count($headerRow) === count($row)) {
|
||||
// Combine the header row with the data row
|
||||
$rawData = array_combine($headerRow, $row);
|
||||
|
||||
// Map the raw data to our model fields
|
||||
$data = [];
|
||||
foreach ($headerMap as $csvField => $modelField) {
|
||||
$data[$modelField] = $rawData[$csvField] ?? null;
|
||||
}
|
||||
|
||||
try {
|
||||
// Skip header row if it was included in the data
|
||||
if ($data['id_category'] !== 'id') {
|
||||
// Use firstOrNew instead of updateOrCreate
|
||||
$category = Category::firstOrNew(['id_category' => $data['id_category']]);
|
||||
$category->fill($data);
|
||||
$category->save();
|
||||
$processedCount++;
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
$errorCount++;
|
||||
Log::error("Error processing Category at row $rowCount in $filePath: " . $e->getMessage());
|
||||
}
|
||||
} else {
|
||||
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headerRow) . ", Got: " . count($row));
|
||||
}
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors.");
|
||||
|
||||
// Clean up the temporary file
|
||||
unlink($tempFilePath);
|
||||
} else {
|
||||
Log::error("Unable to open file: $filePath");
|
||||
}
|
||||
|
||||
Log::info("Category data processing completed. Total processed: $processedCount, Total errors: $errorCount");
|
||||
|
||||
$this->processPeriod();
|
||||
$this->logJobCompletion();
|
||||
} catch (Exception $e) {
|
||||
Log::error('Error in ProcessCategoryDataJob: ' . $e->getMessage());
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
private function initializeJob()
|
||||
: void
|
||||
{
|
||||
set_time_limit(self::MAX_EXECUTION_TIME);
|
||||
$this->processedCount = 0;
|
||||
$this->errorCount = 0;
|
||||
}
|
||||
|
||||
private function processPeriod()
|
||||
: void
|
||||
{
|
||||
$disk = Storage::disk(self::DISK_NAME);
|
||||
$filename = "{$this->period}." . self::FILENAME;
|
||||
$filePath = "{$this->period}/$filename";
|
||||
|
||||
if (!$this->validateFile($disk, $filePath)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$tempFilePath = $this->createTemporaryFile($disk, $filePath, $filename);
|
||||
$this->processFile($tempFilePath, $filePath);
|
||||
$this->cleanup($tempFilePath);
|
||||
}
|
||||
|
||||
private function validateFile($disk, string $filePath)
|
||||
: bool
|
||||
{
|
||||
Log::info("Processing category file: $filePath");
|
||||
|
||||
if (!$disk->exists($filePath)) {
|
||||
Log::warning("File not found: $filePath");
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private function createTemporaryFile($disk, string $filePath, string $filename)
|
||||
: string
|
||||
{
|
||||
$tempFilePath = storage_path("app/temp_$filename");
|
||||
file_put_contents($tempFilePath, $disk->get($filePath));
|
||||
return $tempFilePath;
|
||||
}
|
||||
|
||||
private function processFile(string $tempFilePath, string $filePath)
|
||||
: void
|
||||
{
|
||||
$handle = fopen($tempFilePath, "r");
|
||||
if ($handle === false) {
|
||||
Log::error("Unable to open file: $filePath");
|
||||
return;
|
||||
}
|
||||
|
||||
// Get the headers from the first row
|
||||
$headerRow = fgetcsv($handle, 0, self::CSV_DELIMITER);
|
||||
if (!$headerRow) {
|
||||
fclose($handle);
|
||||
return;
|
||||
}
|
||||
|
||||
$rowCount = 0;
|
||||
while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) {
|
||||
$rowCount++;
|
||||
$this->processRow($headerRow, $row, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors.");
|
||||
}
|
||||
|
||||
private function processRow(array $headerRow, array $row, int $rowCount, string $filePath)
|
||||
: void
|
||||
{
|
||||
if (count($headerRow) !== count($row)) {
|
||||
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " .
|
||||
count($headerRow) . ", Got: " . count($row));
|
||||
return;
|
||||
}
|
||||
|
||||
// Combine the header row with the data row
|
||||
$rawData = array_combine($headerRow, $row);
|
||||
$this->mapAndSaveRecord($rawData, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
private function mapAndSaveRecord(array $rawData, int $rowCount, string $filePath)
|
||||
: void
|
||||
{
|
||||
// Map the raw data to our model fields
|
||||
$data = [];
|
||||
foreach (self::HEADER_MAP as $csvField => $modelField) {
|
||||
$data[$modelField] = $rawData[$csvField] ?? null;
|
||||
}
|
||||
|
||||
// Skip header row if it was included in the data
|
||||
if ($data['id_category'] === 'id') {
|
||||
return;
|
||||
}
|
||||
|
||||
$this->saveRecord($data, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
private function saveRecord(array $data, int $rowCount, string $filePath)
|
||||
: void
|
||||
{
|
||||
try {
|
||||
// Use firstOrNew instead of updateOrCreate
|
||||
$category = Category::firstOrNew(['id_category' => $data['id_category']]);
|
||||
$category->fill($data);
|
||||
$category->save();
|
||||
|
||||
$this->processedCount++;
|
||||
} catch (Exception $e) {
|
||||
$this->errorCount++;
|
||||
Log::error("Error processing Category at row $rowCount in $filePath: " . $e->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private function cleanup(string $tempFilePath)
|
||||
: void
|
||||
{
|
||||
if (file_exists($tempFilePath)) {
|
||||
unlink($tempFilePath);
|
||||
}
|
||||
}
|
||||
|
||||
private function logJobCompletion()
|
||||
: void
|
||||
{
|
||||
Log::info("Category data processing completed. " .
|
||||
"Total processed: {$this->processedCount}, Total errors: {$this->errorCount}");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,16 +16,38 @@
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
protected $period;
|
||||
protected $filename;
|
||||
private const CSV_DELIMITER = '~';
|
||||
private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds
|
||||
private const FILENAME = 'ST.COMPANY.csv';
|
||||
private const DISK_NAME = 'sftpStatement';
|
||||
private const FIELD_MAP = [
|
||||
'id' => null, // Not mapped to model
|
||||
'date_time' => null, // Not mapped to model
|
||||
'company_code' => 'code',
|
||||
'company_name' => 'name',
|
||||
'name_address' => 'address',
|
||||
'mnemonic' => 'mnemonic',
|
||||
'customer_company' => 'customer_company',
|
||||
'customer_mnemonic' => 'customer_mnemonic',
|
||||
'company_group' => 'company_group',
|
||||
'curr_no' => 'curr_no',
|
||||
'co_code' => 'co_code',
|
||||
'l_vendor_atm' => 'l_vendor_atm',
|
||||
'l_vendor_cpc' => 'l_vendor_cpc'
|
||||
];
|
||||
private const BOOLEAN_FIELDS = ['l_vendor_atm', 'l_vendor_cpc'];
|
||||
|
||||
private string $period;
|
||||
private string $filename;
|
||||
private int $processedCount = 0;
|
||||
private int $errorCount = 0;
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
*/
|
||||
public function __construct(string $period = '', string $filename = "ST.COMPANY.csv")
|
||||
public function __construct(string $period = '')
|
||||
{
|
||||
$this->period = $period;
|
||||
$this->filename = $filename;
|
||||
$this->period = $period;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -35,123 +57,154 @@
|
||||
: void
|
||||
{
|
||||
try {
|
||||
set_time_limit(24 * 60 * 60);
|
||||
$disk = Storage::disk('sftpStatement');
|
||||
$processedCount = 0;
|
||||
$errorCount = 0;
|
||||
$this->initializeJob();
|
||||
|
||||
if ($this->period === '') {
|
||||
Log::warning('No period provided for company data processing');
|
||||
return;
|
||||
}
|
||||
|
||||
// Construct the filepath based on the period folder name
|
||||
$fileName = "{$this->period}.{$this->filename}";
|
||||
$filePath = "{$this->period}/$fileName";
|
||||
|
||||
Log::info("Processing company file: $filePath");
|
||||
|
||||
if (!$disk->exists($filePath)) {
|
||||
Log::warning("File not found: $filePath");
|
||||
return;
|
||||
}
|
||||
|
||||
// Create a temporary local copy of the file
|
||||
$tempFilePath = storage_path("app/temp_{$this->filename}");
|
||||
file_put_contents($tempFilePath, $disk->get($filePath));
|
||||
|
||||
$handle = fopen($tempFilePath, "r");
|
||||
|
||||
if ($handle !== false) {
|
||||
// CSV headers from the file
|
||||
$csvHeaders = [
|
||||
'id',
|
||||
'date_time',
|
||||
'company_code',
|
||||
'company_name',
|
||||
'name_address',
|
||||
'mnemonic',
|
||||
'customer_company',
|
||||
'customer_mnemonic',
|
||||
'company_group',
|
||||
'curr_no',
|
||||
'co_code',
|
||||
'l_vendor_atm',
|
||||
'l_vendor_cpc'
|
||||
];
|
||||
|
||||
// Field mapping from CSV to Branch model
|
||||
$fieldMapping = [
|
||||
'company_code' => 'code',
|
||||
'company_name' => 'name',
|
||||
'name_address' => 'address',
|
||||
'mnemonic' => 'mnemonic',
|
||||
'customer_company' => 'customer_company',
|
||||
'customer_mnemonic' => 'customer_mnemonic',
|
||||
'company_group' => 'company_group',
|
||||
'curr_no' => 'curr_no',
|
||||
'co_code' => 'co_code',
|
||||
'l_vendor_atm' => 'l_vendor_atm',
|
||||
'l_vendor_cpc' => 'l_vendor_cpc'
|
||||
];
|
||||
|
||||
$rowCount = 0;
|
||||
|
||||
while (($row = fgetcsv($handle, 0, "~")) !== false) {
|
||||
$rowCount++;
|
||||
|
||||
// Skip header row if it exists
|
||||
if ($rowCount === 1 && (strtolower($row[0]) === 'id' || strtolower($row[2]) === 'company_code')) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (count($csvHeaders) === count($row)) {
|
||||
$csvData = array_combine($csvHeaders, $row);
|
||||
|
||||
// Map CSV data to Branch model fields
|
||||
$branchData = [];
|
||||
foreach ($fieldMapping as $csvField => $modelField) {
|
||||
if (isset($csvData[$csvField])) {
|
||||
// Convert string boolean values to actual booleans for boolean fields
|
||||
if (in_array($modelField, ['l_vendor_atm', 'l_vendor_cpc'])) {
|
||||
$branchData[$modelField] = filter_var($csvData[$csvField], FILTER_VALIDATE_BOOLEAN);
|
||||
} else {
|
||||
$branchData[$modelField] = $csvData[$csvField];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
if (!empty($branchData['code'])) {
|
||||
Branch::updateOrCreate(
|
||||
['code' => $branchData['code']],
|
||||
$branchData
|
||||
);
|
||||
$processedCount++;
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
$errorCount++;
|
||||
Log::error("Error processing Company data at row $rowCount in $filePath: " . $e->getMessage());
|
||||
}
|
||||
} else {
|
||||
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($csvHeaders) . ", Got: " . count($row));
|
||||
}
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors.");
|
||||
|
||||
// Clean up the temporary file
|
||||
unlink($tempFilePath);
|
||||
} else {
|
||||
Log::error("Unable to open file: $filePath");
|
||||
}
|
||||
|
||||
Log::info("Company data processing completed. Total processed: $processedCount, Total errors: $errorCount");
|
||||
|
||||
$this->processPeriod();
|
||||
$this->logJobCompletion();
|
||||
} catch (Exception $e) {
|
||||
Log::error('Error in ProcessCompanyDataJob: ' . $e->getMessage());
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
private function initializeJob()
|
||||
: void
|
||||
{
|
||||
set_time_limit(self::MAX_EXECUTION_TIME);
|
||||
$this->processedCount = 0;
|
||||
$this->errorCount = 0;
|
||||
}
|
||||
|
||||
private function processPeriod()
|
||||
: void
|
||||
{
|
||||
$disk = Storage::disk(self::DISK_NAME);
|
||||
$filename = "{$this->period}." . self::FILENAME;
|
||||
$filePath = "{$this->period}/$filename";
|
||||
|
||||
if (!$this->validateFile($disk, $filePath)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$tempFilePath = $this->createTemporaryFile($disk, $filePath, $filename);
|
||||
$this->processFile($tempFilePath, $filePath);
|
||||
$this->cleanup($tempFilePath);
|
||||
}
|
||||
|
||||
private function validateFile($disk, string $filePath)
|
||||
: bool
|
||||
{
|
||||
Log::info("Processing company file: $filePath");
|
||||
|
||||
if (!$disk->exists($filePath)) {
|
||||
Log::warning("File not found: $filePath");
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private function createTemporaryFile($disk, string $filePath, string $filename = self::FILENAME)
|
||||
: string
|
||||
{
|
||||
$tempFilePath = storage_path("app/temp_$filename");
|
||||
file_put_contents($tempFilePath, $disk->get($filePath));
|
||||
return $tempFilePath;
|
||||
}
|
||||
|
||||
private function processFile(string $tempFilePath, string $filePath)
|
||||
: void
|
||||
{
|
||||
$handle = fopen($tempFilePath, "r");
|
||||
if ($handle === false) {
|
||||
Log::error("Unable to open file: $filePath");
|
||||
return;
|
||||
}
|
||||
|
||||
$rowCount = 0;
|
||||
while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) {
|
||||
$rowCount++;
|
||||
|
||||
// Skip header row if it exists
|
||||
if ($rowCount === 1 && (strtolower($row[0]) === 'id' || strtolower($row[2]) === 'company_code')) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$this->processRow($row, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors.");
|
||||
}
|
||||
|
||||
private function processRow(array $row, int $rowCount, string $filePath)
|
||||
: void
|
||||
{
|
||||
$csvHeaders = array_keys(self::FIELD_MAP);
|
||||
|
||||
if (count($csvHeaders) !== count($row)) {
|
||||
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " .
|
||||
count($csvHeaders) . ", Got: " . count($row));
|
||||
return;
|
||||
}
|
||||
|
||||
$csvData = array_combine($csvHeaders, $row);
|
||||
$this->mapAndSaveRecord($csvData, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
private function mapAndSaveRecord(array $csvData, int $rowCount, string $filePath)
|
||||
: void
|
||||
{
|
||||
// Map CSV data to Branch model fields
|
||||
$branchData = [];
|
||||
foreach (self::FIELD_MAP as $csvField => $modelField) {
|
||||
if ($modelField !== null && isset($csvData[$csvField])) {
|
||||
// Convert string boolean values to actual booleans for boolean fields
|
||||
if (in_array($modelField, self::BOOLEAN_FIELDS)) {
|
||||
$branchData[$modelField] = filter_var($csvData[$csvField], FILTER_VALIDATE_BOOLEAN);
|
||||
} else {
|
||||
$branchData[$modelField] = $csvData[$csvField];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$this->saveRecord($branchData, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
private function saveRecord(array $branchData, int $rowCount, string $filePath)
|
||||
: void
|
||||
{
|
||||
try {
|
||||
if (!empty($branchData['code'])) {
|
||||
Branch::updateOrCreate(
|
||||
['code' => $branchData['code']],
|
||||
$branchData
|
||||
);
|
||||
$this->processedCount++;
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
$this->errorCount++;
|
||||
Log::error("Error processing Company data at row $rowCount in $filePath: " . $e->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private function cleanup(string $tempFilePath)
|
||||
: void
|
||||
{
|
||||
if (file_exists($tempFilePath)) {
|
||||
unlink($tempFilePath);
|
||||
}
|
||||
}
|
||||
|
||||
private function logJobCompletion()
|
||||
: void
|
||||
{
|
||||
Log::info("Company data processing completed. " .
|
||||
"Total processed: {$this->processedCount}, Total errors: {$this->errorCount}");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,82 +16,152 @@
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
protected $period;
|
||||
private const CSV_DELIMITER = '~';
|
||||
private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds
|
||||
private const FILENAME = 'ST.CUSTOMER.csv';
|
||||
private const DISK_NAME = 'sftpStatement';
|
||||
|
||||
private string $period;
|
||||
private int $processedCount = 0;
|
||||
private int $errorCount = 0;
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
*/
|
||||
public function __construct(string $period = '')
|
||||
{
|
||||
$this->period = $period;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the job.
|
||||
*/
|
||||
public function handle()
|
||||
: void
|
||||
{
|
||||
try {
|
||||
set_time_limit(24 * 60 * 60);
|
||||
$disk = Storage::disk('sftpStatement');
|
||||
$processedCount = 0;
|
||||
$errorCount = 0;
|
||||
$this->initializeJob();
|
||||
|
||||
if ($this->period === '') {
|
||||
Log::warning('No period provided for customer data processing');
|
||||
return;
|
||||
}
|
||||
|
||||
// Construct the filename based on the period folder name
|
||||
$filename = "{$this->period}.ST.CUSTOMER.csv";
|
||||
$filePath = "{$this->period}/$filename";
|
||||
|
||||
Log::info("Processing customer file: $filePath");
|
||||
|
||||
if (!$disk->exists($filePath)) {
|
||||
Log::warning("File not found: $filePath");
|
||||
return;
|
||||
}
|
||||
|
||||
// Create a temporary local copy of the file
|
||||
$tempFilePath = storage_path("app/temp_$filename");
|
||||
file_put_contents($tempFilePath, $disk->get($filePath));
|
||||
|
||||
$handle = fopen($tempFilePath, "r");
|
||||
|
||||
if ($handle !== false) {
|
||||
$headers = (new Customer())->getFillable();
|
||||
$rowCount = 0;
|
||||
|
||||
while (($row = fgetcsv($handle, 0, "~")) !== false) {
|
||||
$rowCount++;
|
||||
|
||||
if (count($headers) === count($row)) {
|
||||
$data = array_combine($headers, $row);
|
||||
try {
|
||||
if ($data['customer_code'] !== 'customer_code') {
|
||||
$customer = Customer::firstOrNew(['customer_code' => $data['customer_code']]);
|
||||
$customer->fill($data);
|
||||
$customer->save();
|
||||
$processedCount++;
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
$errorCount++;
|
||||
Log::error("Error processing Customer at row $rowCount in $filePath: " . $e->getMessage());
|
||||
}
|
||||
} else {
|
||||
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row));
|
||||
}
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors.");
|
||||
|
||||
// Clean up the temporary file
|
||||
unlink($tempFilePath);
|
||||
} else {
|
||||
Log::error("Unable to open file: $filePath");
|
||||
}
|
||||
|
||||
Log::info("Customer data processing completed. Total processed: $processedCount, Total errors: $errorCount");
|
||||
|
||||
$this->processPeriod();
|
||||
$this->logJobCompletion();
|
||||
} catch (Exception $e) {
|
||||
Log::error('Error in ProcessCustomerDataJob: ' . $e->getMessage());
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
private function initializeJob()
|
||||
: void
|
||||
{
|
||||
set_time_limit(self::MAX_EXECUTION_TIME);
|
||||
$this->processedCount = 0;
|
||||
$this->errorCount = 0;
|
||||
}
|
||||
|
||||
private function processPeriod()
|
||||
: void
|
||||
{
|
||||
$disk = Storage::disk(self::DISK_NAME);
|
||||
$filename = "{$this->period}." . self::FILENAME;
|
||||
$filePath = "{$this->period}/$filename";
|
||||
|
||||
if (!$this->validateFile($disk, $filePath)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$tempFilePath = $this->createTemporaryFile($disk, $filePath, $filename);
|
||||
$this->processFile($tempFilePath, $filePath);
|
||||
$this->cleanup($tempFilePath);
|
||||
}
|
||||
|
||||
private function validateFile($disk, string $filePath)
|
||||
: bool
|
||||
{
|
||||
Log::info("Processing customer file: $filePath");
|
||||
|
||||
if (!$disk->exists($filePath)) {
|
||||
Log::warning("File not found: $filePath");
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private function createTemporaryFile($disk, string $filePath, string $filename)
|
||||
: string
|
||||
{
|
||||
$tempFilePath = storage_path("app/temp_$filename");
|
||||
file_put_contents($tempFilePath, $disk->get($filePath));
|
||||
return $tempFilePath;
|
||||
}
|
||||
|
||||
private function processFile(string $tempFilePath, string $filePath)
|
||||
: void
|
||||
{
|
||||
$handle = fopen($tempFilePath, "r");
|
||||
if ($handle === false) {
|
||||
Log::error("Unable to open file: $filePath");
|
||||
return;
|
||||
}
|
||||
|
||||
$headers = (new Customer())->getFillable();
|
||||
$rowCount = 0;
|
||||
|
||||
while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) {
|
||||
$rowCount++;
|
||||
$this->processRow($row, $headers, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors.");
|
||||
}
|
||||
|
||||
private function processRow(array $row, array $headers, int $rowCount, string $filePath)
|
||||
: void
|
||||
{
|
||||
if (count($headers) !== count($row)) {
|
||||
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " .
|
||||
count($headers) . ", Got: " . count($row));
|
||||
return;
|
||||
}
|
||||
|
||||
$data = array_combine($headers, $row);
|
||||
$this->saveRecord($data, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
private function saveRecord(array $data, int $rowCount, string $filePath)
|
||||
: void
|
||||
{
|
||||
try {
|
||||
if (isset($data['customer_code']) && $data['customer_code'] !== 'customer_code') {
|
||||
$customer = Customer::firstOrNew(['customer_code' => $data['customer_code']]);
|
||||
$customer->fill($data);
|
||||
$customer->save();
|
||||
$this->processedCount++;
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
$this->errorCount++;
|
||||
Log::error("Error processing Customer at row $rowCount in $filePath: " . $e->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private function cleanup(string $tempFilePath)
|
||||
: void
|
||||
{
|
||||
if (file_exists($tempFilePath)) {
|
||||
unlink($tempFilePath);
|
||||
}
|
||||
}
|
||||
|
||||
private function logJobCompletion()
|
||||
: void
|
||||
{
|
||||
Log::info("Customer data processing completed. " .
|
||||
"Total processed: {$this->processedCount}, Total errors: {$this->errorCount}");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,16 +16,63 @@
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
protected $period;
|
||||
protected $filename;
|
||||
private const CSV_DELIMITER = '~';
|
||||
private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds
|
||||
private const FILENAME = 'ST.DATA.CAPTURE.csv';
|
||||
private const DISK_NAME = 'sftpStatement';
|
||||
private const CSV_HEADERS = [
|
||||
'id',
|
||||
'account_number',
|
||||
'sign',
|
||||
'amount_lcy',
|
||||
'transaction_code',
|
||||
'their_reference',
|
||||
'narrative',
|
||||
'pl_category',
|
||||
'customer_id',
|
||||
'account_officer',
|
||||
'product_category',
|
||||
'value_date',
|
||||
'currency',
|
||||
'amount_fcy',
|
||||
'exchange_rate',
|
||||
'neg_ref_no',
|
||||
'position_type',
|
||||
'our_reference',
|
||||
'reversal_marker',
|
||||
'exposure_date',
|
||||
'currency_market',
|
||||
'iblc_country',
|
||||
'last_version',
|
||||
'otor_version',
|
||||
'department_code',
|
||||
'dealer_desk',
|
||||
'bank_sort_cde',
|
||||
'cheque_number',
|
||||
'accounting_date',
|
||||
'contingent_acct',
|
||||
'cheq_type',
|
||||
'tfs_reference',
|
||||
'accounting_company',
|
||||
'stmt_no',
|
||||
'curr_no',
|
||||
'inputter',
|
||||
'authoriser',
|
||||
'co_code',
|
||||
'date_time'
|
||||
];
|
||||
private const DATE_FIELDS = ['value_date', 'exposure_date', 'accounting_date'];
|
||||
|
||||
private string $period;
|
||||
private int $processedCount = 0;
|
||||
private int $errorCount = 0;
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
*/
|
||||
public function __construct(string $period = '', string $filename = "ST.DATA.CAPTURE.csv")
|
||||
public function __construct(string $period = '')
|
||||
{
|
||||
$this->period = $period;
|
||||
$this->filename = $filename;
|
||||
$this->period = $period;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -35,143 +82,160 @@
|
||||
: void
|
||||
{
|
||||
try {
|
||||
set_time_limit(24 * 60 * 60);
|
||||
$disk = Storage::disk('sftpStatement');
|
||||
$processedCount = 0;
|
||||
$errorCount = 0;
|
||||
$this->initializeJob();
|
||||
|
||||
if ($this->period === '') {
|
||||
Log::warning('No period provided for data capture processing');
|
||||
return;
|
||||
}
|
||||
|
||||
// Construct the filepath based on the period folder name
|
||||
$fileName = "{$this->period}.{$this->filename}";
|
||||
$filePath = "{$this->period}/$fileName";
|
||||
|
||||
Log::info("Processing data capture file: $filePath");
|
||||
|
||||
if (!$disk->exists($filePath)) {
|
||||
Log::warning("File not found: $filePath");
|
||||
return;
|
||||
}
|
||||
|
||||
// Create a temporary local copy of the file
|
||||
$tempFilePath = storage_path("app/temp_{$this->filename}");
|
||||
file_put_contents($tempFilePath, $disk->get($filePath));
|
||||
|
||||
$handle = fopen($tempFilePath, "r");
|
||||
|
||||
if ($handle !== false) {
|
||||
// CSV headers from the file
|
||||
$csvHeaders = [
|
||||
'id',
|
||||
'account_number',
|
||||
'sign',
|
||||
'amount_lcy',
|
||||
'transaction_code',
|
||||
'their_reference',
|
||||
'narrative',
|
||||
'pl_category',
|
||||
'customer_id',
|
||||
'account_officer',
|
||||
'product_category',
|
||||
'value_date',
|
||||
'currency',
|
||||
'amount_fcy',
|
||||
'exchange_rate',
|
||||
'neg_ref_no',
|
||||
'position_type',
|
||||
'our_reference',
|
||||
'reversal_marker',
|
||||
'exposure_date',
|
||||
'currency_market',
|
||||
'iblc_country',
|
||||
'last_version',
|
||||
'otor_version',
|
||||
'department_code',
|
||||
'dealer_desk',
|
||||
'bank_sort_cde',
|
||||
'cheque_number',
|
||||
'accounting_date',
|
||||
'contingent_acct',
|
||||
'cheq_type',
|
||||
'tfs_reference',
|
||||
'accounting_company',
|
||||
'stmt_no',
|
||||
'curr_no',
|
||||
'inputter',
|
||||
'authoriser',
|
||||
'co_code',
|
||||
'date_time'
|
||||
];
|
||||
|
||||
$rowCount = 0;
|
||||
|
||||
while (($row = fgetcsv($handle, 0, "~")) !== false) {
|
||||
$rowCount++;
|
||||
|
||||
// Skip header row if it exists
|
||||
if ($rowCount === 1 && strtolower($row[0]) === 'id') {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (count($csvHeaders) === count($row)) {
|
||||
$data = array_combine($csvHeaders, $row);
|
||||
|
||||
try {
|
||||
// Format dates if they exist
|
||||
foreach (['value_date', 'exposure_date', 'accounting_date'] as $dateField) {
|
||||
if (!empty($data[$dateField])) {
|
||||
try {
|
||||
$data[$dateField] = date('Y-m-d', strtotime($data[$dateField]));
|
||||
} catch (Exception $e) {
|
||||
// If date parsing fails, keep the original value
|
||||
Log::warning("Failed to parse date for $dateField: {$data[$dateField]}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Format datetime if it exists
|
||||
if (!empty($data['date_time'])) {
|
||||
try {
|
||||
$data['date_time'] = date('Y-m-d H:i:s', strtotime($data['date_time']));
|
||||
} catch (Exception $e) {
|
||||
// If datetime parsing fails, keep the original value
|
||||
Log::warning("Failed to parse datetime for date_time: {$data['date_time']}");
|
||||
}
|
||||
}
|
||||
|
||||
if (!empty($data['id'])) {
|
||||
DataCapture::updateOrCreate(
|
||||
['id' => $data['id']],
|
||||
$data
|
||||
);
|
||||
$processedCount++;
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
$errorCount++;
|
||||
Log::error("Error processing Data Capture at row $rowCount in $filePath: " . $e->getMessage());
|
||||
}
|
||||
} else {
|
||||
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($csvHeaders) . ", Got: " . count($row));
|
||||
}
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors.");
|
||||
|
||||
// Clean up the temporary file
|
||||
unlink($tempFilePath);
|
||||
} else {
|
||||
Log::error("Unable to open file: $filePath");
|
||||
}
|
||||
|
||||
Log::info("Data capture processing completed. Total processed: $processedCount, Total errors: $errorCount");
|
||||
|
||||
$this->processPeriod();
|
||||
$this->logJobCompletion();
|
||||
} catch (Exception $e) {
|
||||
Log::error('Error in ProcessDataCaptureDataJob: ' . $e->getMessage());
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
private function initializeJob()
|
||||
: void
|
||||
{
|
||||
set_time_limit(self::MAX_EXECUTION_TIME);
|
||||
$this->processedCount = 0;
|
||||
$this->errorCount = 0;
|
||||
}
|
||||
|
||||
private function processPeriod()
|
||||
: void
|
||||
{
|
||||
$disk = Storage::disk(self::DISK_NAME);
|
||||
$filename = "{$this->period}." . self::FILENAME;
|
||||
$filePath = "{$this->period}/$filename";
|
||||
|
||||
if (!$this->validateFile($disk, $filePath)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$tempFilePath = $this->createTemporaryFile($disk, $filePath, $filename);
|
||||
$this->processFile($tempFilePath, $filePath);
|
||||
$this->cleanup($tempFilePath);
|
||||
}
|
||||
|
||||
private function validateFile($disk, string $filePath)
|
||||
: bool
|
||||
{
|
||||
Log::info("Processing data capture file: $filePath");
|
||||
|
||||
if (!$disk->exists($filePath)) {
|
||||
Log::warning("File not found: $filePath");
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private function createTemporaryFile($disk, string $filePath, string $filename)
|
||||
: string
|
||||
{
|
||||
$tempFilePath = storage_path("app/temp_$filename");
|
||||
file_put_contents($tempFilePath, $disk->get($filePath));
|
||||
return $tempFilePath;
|
||||
}
|
||||
|
||||
private function processFile(string $tempFilePath, string $filePath)
|
||||
: void
|
||||
{
|
||||
$handle = fopen($tempFilePath, "r");
|
||||
if ($handle === false) {
|
||||
Log::error("Unable to open file: $filePath");
|
||||
return;
|
||||
}
|
||||
|
||||
$rowCount = 0;
|
||||
while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) {
|
||||
$rowCount++;
|
||||
|
||||
// Skip header row if it exists
|
||||
if ($rowCount === 1 && strtolower($row[0]) === 'id') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$this->processRow($row, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors.");
|
||||
}
|
||||
|
||||
private function processRow(array $row, int $rowCount, string $filePath)
|
||||
: void
|
||||
{
|
||||
if (count(self::CSV_HEADERS) !== count($row)) {
|
||||
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " .
|
||||
count(self::CSV_HEADERS) . ", Got: " . count($row));
|
||||
return;
|
||||
}
|
||||
|
||||
$data = array_combine(self::CSV_HEADERS, $row);
|
||||
$this->formatDates($data);
|
||||
$this->saveRecord($data, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
private function formatDates(array &$data)
|
||||
: void
|
||||
{
|
||||
// Format dates if they exist
|
||||
foreach (self::DATE_FIELDS as $dateField) {
|
||||
if (!empty($data[$dateField])) {
|
||||
try {
|
||||
$data[$dateField] = date('Y-m-d', strtotime($data[$dateField]));
|
||||
} catch (Exception $e) {
|
||||
// If date parsing fails, keep the original value
|
||||
Log::warning("Failed to parse date for $dateField: {$data[$dateField]}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Format datetime if it exists
|
||||
if (!empty($data['date_time'])) {
|
||||
try {
|
||||
$data['date_time'] = date('Y-m-d H:i:s', strtotime($data['date_time']));
|
||||
} catch (Exception $e) {
|
||||
// If datetime parsing fails, keep the original value
|
||||
Log::warning("Failed to parse datetime for date_time: {$data['date_time']}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private function saveRecord(array $data, int $rowCount, string $filePath)
|
||||
: void
|
||||
{
|
||||
try {
|
||||
if (!empty($data['id'])) {
|
||||
DataCapture::updateOrCreate(
|
||||
['id' => $data['id']],
|
||||
$data
|
||||
);
|
||||
$this->processedCount++;
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
$this->errorCount++;
|
||||
Log::error("Error processing Data Capture at row $rowCount in $filePath: " . $e->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private function cleanup(string $tempFilePath)
|
||||
: void
|
||||
{
|
||||
if (file_exists($tempFilePath)) {
|
||||
unlink($tempFilePath);
|
||||
}
|
||||
}
|
||||
|
||||
private function logJobCompletion()
|
||||
: void
|
||||
{
|
||||
Log::info("Data capture processing completed. " .
|
||||
"Total processed: {$this->processedCount}, Total errors: {$this->errorCount}");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,113 +1,175 @@
|
||||
<?php
|
||||
|
||||
namespace Modules\Webstatement\Jobs;
|
||||
namespace Modules\Webstatement\Jobs;
|
||||
|
||||
use Exception;
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use Modules\Webstatement\Models\FtTxnTypeCondition;
|
||||
use Exception;
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use Modules\Webstatement\Models\FtTxnTypeCondition;
|
||||
|
||||
class ProcessFtTxnTypeConditionJob implements ShouldQueue
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
protected $filename;
|
||||
protected array $periods;
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
*/
|
||||
public function __construct(array|string $periods = [])
|
||||
class ProcessFtTxnTypeConditionJob implements ShouldQueue
|
||||
{
|
||||
$this->periods = is_string($periods) ? [$periods] : $periods;
|
||||
}
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
private const CSV_DELIMITER = '~';
|
||||
private const EXPECTED_HEADERS = [
|
||||
'id',
|
||||
'date_time',
|
||||
'transaction_type',
|
||||
'short_descr',
|
||||
'txn_code_cr',
|
||||
'txn_code_dr'
|
||||
];
|
||||
private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds
|
||||
private const FILENAME = 'ST.FT.TXN.TYPE.CONDITION.csv';
|
||||
private const DISK_NAME = 'sftpStatement';
|
||||
|
||||
private string $period;
|
||||
private int $processedCount = 0;
|
||||
private int $errorCount = 0;
|
||||
|
||||
public function __construct(string $period = '')
|
||||
{
|
||||
$this->period = $period;
|
||||
}
|
||||
|
||||
public function handle()
|
||||
: void
|
||||
{
|
||||
try {
|
||||
$this->initializeJob();
|
||||
|
||||
if ($this->period === '') {
|
||||
Log::warning('No periods provided for transaction type condition data processing');
|
||||
return;
|
||||
}
|
||||
|
||||
$this->processPeriod();
|
||||
$this->logJobCompletion();
|
||||
} catch (Exception $e) {
|
||||
Log::error('Error in ProcessFtTxnTypeConditionJob: ' . $e->getMessage());
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
private function initializeJob()
|
||||
: void
|
||||
{
|
||||
set_time_limit(self::MAX_EXECUTION_TIME);
|
||||
$this->processedCount = 0;
|
||||
$this->errorCount = 0;
|
||||
}
|
||||
|
||||
private function processPeriod()
|
||||
: void
|
||||
{
|
||||
$disk = Storage::disk(self::DISK_NAME);
|
||||
$filePath = "$this->period/" . self::FILENAME;
|
||||
|
||||
|
||||
/**
|
||||
* Execute the job.
|
||||
*/
|
||||
public function handle(): void
|
||||
{
|
||||
try {
|
||||
set_time_limit(24 * 60 * 60);
|
||||
$disk = Storage::disk('sftpStatement');
|
||||
$processedCount = 0;
|
||||
$errorCount = 0;
|
||||
|
||||
if (empty($this->periods)) {
|
||||
Log::warning('No periods provided for transaction type condition data processing');
|
||||
if (!$this->validateFile($disk, $filePath)) {
|
||||
return;
|
||||
}
|
||||
|
||||
foreach ($this->periods as $period) {
|
||||
$tempFilePath = $this->createTemporaryFile($disk, $filePath);
|
||||
$this->processFile($tempFilePath, $filePath);
|
||||
$this->cleanup($tempFilePath);
|
||||
}
|
||||
|
||||
// Construct the filepath based on the period folder name
|
||||
$filePath = "$period/{$this->filename}";
|
||||
private function validateFile($disk, string $filePath)
|
||||
: bool
|
||||
{
|
||||
Log::info("Processing transaction type condition file: $filePath");
|
||||
|
||||
Log::info("Processing transaction type condition file: $filePath");
|
||||
if (!$disk->exists($filePath)) {
|
||||
Log::warning("File not found: $filePath");
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!$disk->exists($filePath)) {
|
||||
Log::warning("File not found: $filePath");
|
||||
return true;
|
||||
}
|
||||
|
||||
private function createTemporaryFile($disk, string $filePath)
|
||||
: string
|
||||
{
|
||||
$tempFilePath = storage_path("app/temp_" . self::FILENAME);
|
||||
file_put_contents($tempFilePath, $disk->get($filePath));
|
||||
return $tempFilePath;
|
||||
}
|
||||
|
||||
private function processFile(string $tempFilePath, string $filePath)
|
||||
: void
|
||||
{
|
||||
$handle = fopen($tempFilePath, "r");
|
||||
if ($handle === false) {
|
||||
Log::error("Unable to open file: $filePath");
|
||||
return;
|
||||
}
|
||||
|
||||
$rowCount = 0;
|
||||
while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) {
|
||||
$rowCount++;
|
||||
if ($this->isHeaderRow($rowCount, $row)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Create a temporary local copy of the file
|
||||
$tempFilePath = storage_path("app/temp_{$this->filename}");
|
||||
file_put_contents($tempFilePath, $disk->get($filePath));
|
||||
|
||||
$handle = fopen($tempFilePath, "r");
|
||||
|
||||
if ($handle !== false) {
|
||||
$headers = ['id', 'date_time', 'transaction_type', 'short_descr', 'txn_code_cr', 'txn_code_dr'];
|
||||
$rowCount = 0;
|
||||
|
||||
while (($row = fgetcsv($handle, 0, "~")) !== false) {
|
||||
$rowCount++;
|
||||
|
||||
// Skip header row if it exists
|
||||
if ($rowCount === 1 && strtolower($row[0]) === 'id') {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (count($headers) === count($row)) {
|
||||
$data = array_combine($headers, $row);
|
||||
try {
|
||||
if (!empty($data['id'])) {
|
||||
FtTxnTypeCondition::updateOrCreate(
|
||||
['id' => $data['id']],
|
||||
$data
|
||||
);
|
||||
$processedCount++;
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
$errorCount++;
|
||||
Log::error("Error processing Transaction Type Condition at row $rowCount in $filePath: " . $e->getMessage());
|
||||
}
|
||||
} else {
|
||||
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row));
|
||||
}
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors.");
|
||||
|
||||
// Clean up the temporary file
|
||||
unlink($tempFilePath);
|
||||
} else {
|
||||
Log::error("Unable to open file: $filePath");
|
||||
}
|
||||
$this->processRow($row, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
Log::info("Transaction type condition data processing completed. Total processed: $processedCount, Total errors: $errorCount");
|
||||
fclose($handle);
|
||||
Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors.");
|
||||
}
|
||||
|
||||
} catch (Exception $e) {
|
||||
Log::error('Error in ProcessFtTxnTypeConditionJob: ' . $e->getMessage());
|
||||
throw $e;
|
||||
private function isHeaderRow(int $rowCount, array $row)
|
||||
: bool
|
||||
{
|
||||
return $rowCount === 1 && strtolower($row[0]) === 'id';
|
||||
}
|
||||
|
||||
private function processRow(array $row, int $rowCount, string $filePath)
|
||||
: void
|
||||
{
|
||||
if (count(self::EXPECTED_HEADERS) !== count($row)) {
|
||||
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " .
|
||||
count(self::EXPECTED_HEADERS) . ", Got: " . count($row));
|
||||
return;
|
||||
}
|
||||
|
||||
$data = array_combine(self::EXPECTED_HEADERS, $row);
|
||||
$this->saveRecord($data, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
private function saveRecord(array $data, int $rowCount, string $filePath)
|
||||
: void
|
||||
{
|
||||
try {
|
||||
if (!empty($data['id'])) {
|
||||
FtTxnTypeCondition::updateOrCreate(['id' => $data['id']], $data);
|
||||
$this->processedCount++;
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
$this->errorCount++;
|
||||
Log::error("Error processing Transaction Type Condition at row $rowCount in $filePath: " . $e->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private function cleanup(string $tempFilePath)
|
||||
: void
|
||||
{
|
||||
if (file_exists($tempFilePath)) {
|
||||
unlink($tempFilePath);
|
||||
}
|
||||
}
|
||||
|
||||
private function logJobCompletion()
|
||||
: void
|
||||
{
|
||||
Log::info("Transaction type condition data processing completed. " .
|
||||
"Total processed: {$this->processedCount}, Total errors: {$this->errorCount}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,7 +16,14 @@
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
protected $period;
|
||||
private const CSV_DELIMITER = '~';
|
||||
private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds
|
||||
private const FILENAME = 'ST.FUNDS.TRANSFER.csv';
|
||||
private const DISK_NAME = 'sftpStatement';
|
||||
|
||||
private string $period;
|
||||
private int $processedCount = 0;
|
||||
private int $errorCount = 0;
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
@@ -33,78 +40,134 @@
|
||||
: void
|
||||
{
|
||||
try {
|
||||
set_time_limit(24 * 60 * 60);
|
||||
$disk = Storage::disk('sftpStatement');
|
||||
$processedCount = 0;
|
||||
$errorCount = 0;
|
||||
$this->initializeJob();
|
||||
|
||||
if ($this->period === '') {
|
||||
Log::warning('No period provided for funds transfer data processing');
|
||||
return;
|
||||
}
|
||||
|
||||
// Construct the filename based on the period folder name
|
||||
$filename = "{$this->period}.ST.FUNDS.TRANSFER.csv";
|
||||
$filePath = "{$this->period}/$filename";
|
||||
|
||||
Log::info("Processing funds transfer file: $filePath");
|
||||
|
||||
if (!$disk->exists($filePath)) {
|
||||
Log::warning("File not found: $filePath");
|
||||
return;
|
||||
}
|
||||
|
||||
// Create a temporary local copy of the file
|
||||
$tempFilePath = storage_path("app/temp_$filename");
|
||||
file_put_contents($tempFilePath, $disk->get($filePath));
|
||||
|
||||
$handle = fopen($tempFilePath, "r");
|
||||
|
||||
if ($handle !== false) {
|
||||
$headers = (new TempFundsTransfer())->getFillable();
|
||||
$rowCount = 0;
|
||||
|
||||
while (($row = fgetcsv($handle, 0, "~")) !== false) {
|
||||
$rowCount++;
|
||||
|
||||
// Handle case where row has more columns than headers
|
||||
if (count($row) > count($headers)) {
|
||||
$row = array_slice($row, 0, count($headers));
|
||||
}
|
||||
|
||||
if (count($headers) === count($row)) {
|
||||
$data = array_combine($headers, $row);
|
||||
try {
|
||||
if (isset($data['_id']) && $data['_id'] !== '_id') {
|
||||
TempFundsTransfer::updateOrCreate(
|
||||
['_id' => $data['_id']],
|
||||
$data
|
||||
);
|
||||
$processedCount++;
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
$errorCount++;
|
||||
Log::error("Error processing Funds Transfer at row $rowCount in $filePath: " . $e->getMessage());
|
||||
}
|
||||
} else {
|
||||
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row));
|
||||
}
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors.");
|
||||
|
||||
// Clean up the temporary file
|
||||
unlink($tempFilePath);
|
||||
} else {
|
||||
Log::error("Unable to open file: $filePath");
|
||||
}
|
||||
|
||||
Log::info("Funds Transfer data processing completed. Total processed: $processedCount, Total errors: $errorCount");
|
||||
|
||||
$this->processPeriod();
|
||||
$this->logJobCompletion();
|
||||
} catch (Exception $e) {
|
||||
Log::error('Error in ProcessFundsTransferDataJob: ' . $e->getMessage());
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
private function initializeJob()
|
||||
: void
|
||||
{
|
||||
set_time_limit(self::MAX_EXECUTION_TIME);
|
||||
$this->processedCount = 0;
|
||||
$this->errorCount = 0;
|
||||
}
|
||||
|
||||
private function processPeriod()
|
||||
: void
|
||||
{
|
||||
$disk = Storage::disk(self::DISK_NAME);
|
||||
$filename = "{$this->period}." . self::FILENAME;
|
||||
$filePath = "{$this->period}/$filename";
|
||||
|
||||
if (!$this->validateFile($disk, $filePath)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$tempFilePath = $this->createTemporaryFile($disk, $filePath, $filename);
|
||||
$this->processFile($tempFilePath, $filePath);
|
||||
$this->cleanup($tempFilePath);
|
||||
}
|
||||
|
||||
private function validateFile($disk, string $filePath)
|
||||
: bool
|
||||
{
|
||||
Log::info("Processing funds transfer file: $filePath");
|
||||
|
||||
if (!$disk->exists($filePath)) {
|
||||
Log::warning("File not found: $filePath");
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private function createTemporaryFile($disk, string $filePath, string $filename)
|
||||
: string
|
||||
{
|
||||
$tempFilePath = storage_path("app/temp_$filename");
|
||||
file_put_contents($tempFilePath, $disk->get($filePath));
|
||||
return $tempFilePath;
|
||||
}
|
||||
|
||||
private function processFile(string $tempFilePath, string $filePath)
|
||||
: void
|
||||
{
|
||||
$handle = fopen($tempFilePath, "r");
|
||||
if ($handle === false) {
|
||||
Log::error("Unable to open file: $filePath");
|
||||
return;
|
||||
}
|
||||
|
||||
$headers = (new TempFundsTransfer())->getFillable();
|
||||
$rowCount = 0;
|
||||
|
||||
while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) {
|
||||
$rowCount++;
|
||||
$this->processRow($row, $headers, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors.");
|
||||
}
|
||||
|
||||
private function processRow(array $row, array $headers, int $rowCount, string $filePath)
|
||||
: void
|
||||
{
|
||||
// Handle case where row has more columns than headers
|
||||
if (count($row) > count($headers)) {
|
||||
$row = array_slice($row, 0, count($headers));
|
||||
}
|
||||
|
||||
if (count($headers) !== count($row)) {
|
||||
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " .
|
||||
count($headers) . ", Got: " . count($row));
|
||||
return;
|
||||
}
|
||||
|
||||
$data = array_combine($headers, $row);
|
||||
$this->saveRecord($data, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
private function saveRecord(array $data, int $rowCount, string $filePath)
|
||||
: void
|
||||
{
|
||||
try {
|
||||
if (isset($data['_id']) && $data['_id'] !== '_id') {
|
||||
TempFundsTransfer::updateOrCreate(
|
||||
['_id' => $data['_id']],
|
||||
$data
|
||||
);
|
||||
$this->processedCount++;
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
$this->errorCount++;
|
||||
Log::error("Error processing Funds Transfer at row $rowCount in $filePath: " . $e->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private function cleanup(string $tempFilePath)
|
||||
: void
|
||||
{
|
||||
if (file_exists($tempFilePath)) {
|
||||
unlink($tempFilePath);
|
||||
}
|
||||
}
|
||||
|
||||
private function logJobCompletion()
|
||||
: void
|
||||
{
|
||||
Log::info("Funds Transfer data processing completed. " .
|
||||
"Total processed: {$this->processedCount}, Total errors: {$this->errorCount}");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,7 +16,14 @@
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
protected $period;
|
||||
private const CSV_DELIMITER = '~';
|
||||
private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds
|
||||
private const FILENAME = 'ST.STMT.ENTRY.csv';
|
||||
private const DISK_NAME = 'sftpStatement';
|
||||
|
||||
private string $period;
|
||||
private int $processedCount = 0;
|
||||
private int $errorCount = 0;
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
@@ -33,84 +40,139 @@
|
||||
: void
|
||||
{
|
||||
try {
|
||||
set_time_limit(24 * 60 * 60);
|
||||
$disk = Storage::disk('sftpStatement');
|
||||
$processedCount = 0;
|
||||
$errorCount = 0;
|
||||
$this->initializeJob();
|
||||
|
||||
if ($this->period === '') {
|
||||
Log::warning('No period provided for statement entry data processing');
|
||||
return;
|
||||
}
|
||||
|
||||
// Construct the filename based on the period folder name
|
||||
$filename = "{$this->period}.ST.STMT.ENTRY.csv";
|
||||
$filePath = "{$this->period}/$filename";
|
||||
|
||||
Log::info("Processing statement entry file: $filePath");
|
||||
|
||||
if (!$disk->exists($filePath)) {
|
||||
Log::warning("File not found: $filePath");
|
||||
return;
|
||||
}
|
||||
|
||||
// Create a temporary local copy of the file
|
||||
$tempFilePath = storage_path("app/temp_$filename");
|
||||
file_put_contents($tempFilePath, $disk->get($filePath));
|
||||
|
||||
$handle = fopen($tempFilePath, "r");
|
||||
|
||||
if ($handle !== false) {
|
||||
$headers = (new StmtEntry())->getFillable();
|
||||
$rowCount = 0;
|
||||
|
||||
while (($row = fgetcsv($handle, 0, "~")) !== false) {
|
||||
$rowCount++;
|
||||
|
||||
if (count($headers) === count($row)) {
|
||||
$data = array_combine($headers, $row);
|
||||
try {
|
||||
if ($data['stmt_entry_id'] !== 'stmt_entry_id') {
|
||||
// Bersihkan trans_reference dari \\BNK jika ada
|
||||
if (isset($data['trans_reference'])) {
|
||||
$cleanedRef = preg_replace('/\\\\.*$/', '', $data['trans_reference']);
|
||||
//$firstTwoChars = substr($cleanedRef, 0, 2);
|
||||
/*$data['trans_reference'] = (in_array($firstTwoChars, ['FT', 'TT','AA']))
|
||||
? substr($cleanedRef, 0, 12)
|
||||
: $cleanedRef;*/
|
||||
|
||||
$data['trans_reference'] = $cleanedRef;
|
||||
}
|
||||
|
||||
StmtEntry::updateOrCreate(
|
||||
['stmt_entry_id' => $data['stmt_entry_id']],
|
||||
$data
|
||||
);
|
||||
$processedCount++;
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
$errorCount++;
|
||||
Log::error("Error processing Statement Entry at row $rowCount in $filePath: " . $e->getMessage());
|
||||
}
|
||||
} else {
|
||||
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row));
|
||||
}
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors.");
|
||||
|
||||
// Clean up the temporary file
|
||||
unlink($tempFilePath);
|
||||
} else {
|
||||
Log::error("Unable to open file: $filePath");
|
||||
}
|
||||
|
||||
Log::info("Statement Entry data processing completed. Total processed: $processedCount, Total errors: $errorCount");
|
||||
|
||||
$this->processPeriod();
|
||||
$this->logJobCompletion();
|
||||
} catch (Exception $e) {
|
||||
Log::error('Error in ProcessStmtEntryDataJob: ' . $e->getMessage());
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
private function initializeJob()
|
||||
: void
|
||||
{
|
||||
set_time_limit(self::MAX_EXECUTION_TIME);
|
||||
$this->processedCount = 0;
|
||||
$this->errorCount = 0;
|
||||
}
|
||||
|
||||
private function processPeriod()
|
||||
: void
|
||||
{
|
||||
$disk = Storage::disk(self::DISK_NAME);
|
||||
$filename = "{$this->period}." . self::FILENAME;
|
||||
$filePath = "{$this->period}/$filename";
|
||||
|
||||
if (!$this->validateFile($disk, $filePath)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$tempFilePath = $this->createTemporaryFile($disk, $filePath, $filename);
|
||||
$this->processFile($tempFilePath, $filePath);
|
||||
$this->cleanup($tempFilePath);
|
||||
}
|
||||
|
||||
private function validateFile($disk, string $filePath)
|
||||
: bool
|
||||
{
|
||||
Log::info("Processing statement entry file: $filePath");
|
||||
|
||||
if (!$disk->exists($filePath)) {
|
||||
Log::warning("File not found: $filePath");
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private function createTemporaryFile($disk, string $filePath, string $filename)
|
||||
: string
|
||||
{
|
||||
$tempFilePath = storage_path("app/temp_$filename");
|
||||
file_put_contents($tempFilePath, $disk->get($filePath));
|
||||
return $tempFilePath;
|
||||
}
|
||||
|
||||
private function processFile(string $tempFilePath, string $filePath)
|
||||
: void
|
||||
{
|
||||
$handle = fopen($tempFilePath, "r");
|
||||
if ($handle === false) {
|
||||
Log::error("Unable to open file: $filePath");
|
||||
return;
|
||||
}
|
||||
|
||||
$headers = (new StmtEntry())->getFillable();
|
||||
$rowCount = 0;
|
||||
|
||||
while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) {
|
||||
$rowCount++;
|
||||
$this->processRow($row, $headers, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors.");
|
||||
}
|
||||
|
||||
private function processRow(array $row, array $headers, int $rowCount, string $filePath)
|
||||
: void
|
||||
{
|
||||
if (count($headers) !== count($row)) {
|
||||
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " .
|
||||
count($headers) . ", Got: " . count($row));
|
||||
return;
|
||||
}
|
||||
|
||||
$data = array_combine($headers, $row);
|
||||
$this->cleanTransReference($data);
|
||||
$this->saveRecord($data, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
private function cleanTransReference(array &$data)
|
||||
: void
|
||||
{
|
||||
if (isset($data['trans_reference'])) {
|
||||
// Clean trans_reference from \\BNK if present
|
||||
$data['trans_reference'] = preg_replace('/\\\\.*$/', '', $data['trans_reference']);
|
||||
}
|
||||
}
|
||||
|
||||
private function saveRecord(array $data, int $rowCount, string $filePath)
|
||||
: void
|
||||
{
|
||||
try {
|
||||
if (isset($data['stmt_entry_id']) && $data['stmt_entry_id'] !== 'stmt_entry_id') {
|
||||
StmtEntry::updateOrCreate(
|
||||
['stmt_entry_id' => $data['stmt_entry_id']],
|
||||
$data
|
||||
);
|
||||
$this->processedCount++;
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
$this->errorCount++;
|
||||
Log::error("Error processing Statement Entry at row $rowCount in $filePath: " . $e->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private function cleanup(string $tempFilePath)
|
||||
: void
|
||||
{
|
||||
if (file_exists($tempFilePath)) {
|
||||
unlink($tempFilePath);
|
||||
}
|
||||
}
|
||||
|
||||
private function logJobCompletion()
|
||||
: void
|
||||
{
|
||||
Log::info("Statement Entry data processing completed. " .
|
||||
"Total processed: {$this->processedCount}, Total errors: {$this->errorCount}");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,113 +1,164 @@
|
||||
<?php
|
||||
|
||||
namespace Modules\Webstatement\Jobs;
|
||||
namespace Modules\Webstatement\Jobs;
|
||||
|
||||
use Exception;
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use Modules\Webstatement\Models\TempStmtNarrFormat;
|
||||
use Exception;
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use Modules\Webstatement\Models\TempStmtNarrFormat;
|
||||
|
||||
class ProcessStmtNarrFormatDataJob implements ShouldQueue
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
protected array $periods;
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
*/
|
||||
public function __construct(array|string $periods = [])
|
||||
class ProcessStmtNarrFormatDataJob implements ShouldQueue
|
||||
{
|
||||
$this->periods = is_string($periods) ? [$periods] : $periods;
|
||||
}
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
private const CSV_DELIMITER = '~';
|
||||
private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds
|
||||
private const FILENAME = 'ST.STMT.NARR.FORMAT.csv';
|
||||
private const DISK_NAME = 'sftpStatement';
|
||||
|
||||
/**
|
||||
* Execute the job.
|
||||
*/
|
||||
public function handle(): void
|
||||
{
|
||||
try {
|
||||
set_time_limit(24 * 60 * 60);
|
||||
$disk = Storage::disk('sftpStatement');
|
||||
$processedCount = 0;
|
||||
$errorCount = 0;
|
||||
private string $period;
|
||||
private int $processedCount = 0;
|
||||
private int $errorCount = 0;
|
||||
|
||||
if (empty($this->periods)) {
|
||||
Log::warning('No periods provided for statement narrative format data processing');
|
||||
/**
|
||||
* Create a new job instance.
|
||||
*/
|
||||
public function __construct(string $period = '')
|
||||
{
|
||||
$this->period = $period;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the job.
|
||||
*/
|
||||
public function handle()
|
||||
: void
|
||||
{
|
||||
try {
|
||||
$this->initializeJob();
|
||||
|
||||
if ($this->period === '') {
|
||||
Log::warning('No periods provided for statement narrative format data processing');
|
||||
return;
|
||||
}
|
||||
|
||||
$this->processPeriod();
|
||||
$this->logJobCompletion();
|
||||
} catch (Exception $e) {
|
||||
Log::error('Error in ProcessStmtNarrFormatDataJob: ' . $e->getMessage());
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
private function initializeJob()
|
||||
: void
|
||||
{
|
||||
set_time_limit(self::MAX_EXECUTION_TIME);
|
||||
$this->processedCount = 0;
|
||||
$this->errorCount = 0;
|
||||
}
|
||||
|
||||
private function processPeriod()
|
||||
: void
|
||||
{
|
||||
$disk = Storage::disk(self::DISK_NAME);
|
||||
$filePath = "$this->period/" . self::FILENAME;
|
||||
|
||||
if (!$this->validateFile($disk, $filePath)) {
|
||||
return;
|
||||
}
|
||||
|
||||
foreach ($this->periods as $period) {
|
||||
// Skip the _parameter folder
|
||||
/*if ($period === '_parameter') {
|
||||
Log::info("Skipping _parameter folder");
|
||||
continue;
|
||||
}*/
|
||||
$tempFilePath = $this->createTemporaryFile($disk, $filePath);
|
||||
$this->processFile($tempFilePath, $filePath);
|
||||
$this->cleanup($tempFilePath);
|
||||
}
|
||||
|
||||
// Construct the filename based on the period folder name
|
||||
$filename = "ST.STMT.NARR.FORMAT.csv";
|
||||
$filePath = "$period/$filename";
|
||||
private function validateFile($disk, string $filePath)
|
||||
: bool
|
||||
{
|
||||
Log::info("Processing statement narrative format file: $filePath");
|
||||
|
||||
Log::info("Processing statement narrative format file: $filePath");
|
||||
|
||||
if (!$disk->exists($filePath)) {
|
||||
Log::warning("File not found: $filePath");
|
||||
continue;
|
||||
}
|
||||
|
||||
// Create a temporary local copy of the file
|
||||
$tempFilePath = storage_path("app/temp_$filename");
|
||||
file_put_contents($tempFilePath, $disk->get($filePath));
|
||||
|
||||
$handle = fopen($tempFilePath, "r");
|
||||
|
||||
if ($handle !== false) {
|
||||
$headers = (new TempStmtNarrFormat())->getFillable();
|
||||
$rowCount = 0;
|
||||
|
||||
while (($row = fgetcsv($handle, 0, "~")) !== false) {
|
||||
$rowCount++;
|
||||
|
||||
if (count($headers) === count($row)) {
|
||||
$data = array_combine($headers, $row);
|
||||
try {
|
||||
if (isset($data['_id']) && $data['_id'] !== 'id' && $data['_id'] !== '_id') {
|
||||
TempStmtNarrFormat::updateOrCreate(
|
||||
['_id' => $data['_id']],
|
||||
$data
|
||||
);
|
||||
$processedCount++;
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
$errorCount++;
|
||||
Log::error("Error processing Statement Narrative Format at row $rowCount in $filePath: " . $e->getMessage());
|
||||
}
|
||||
} else {
|
||||
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row));
|
||||
}
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors.");
|
||||
|
||||
// Clean up the temporary file
|
||||
unlink($tempFilePath);
|
||||
} else {
|
||||
Log::error("Unable to open file: $filePath");
|
||||
}
|
||||
if (!$disk->exists($filePath)) {
|
||||
Log::warning("File not found: $filePath");
|
||||
return false;
|
||||
}
|
||||
|
||||
Log::info("Statement Narrative Format data processing completed. Total processed: $processedCount, Total errors: $errorCount");
|
||||
return true;
|
||||
}
|
||||
|
||||
} catch (Exception $e) {
|
||||
Log::error('Error in ProcessStmtNarrFormatDataJob: ' . $e->getMessage());
|
||||
throw $e;
|
||||
private function createTemporaryFile($disk, string $filePath)
|
||||
: string
|
||||
{
|
||||
$tempFilePath = storage_path("app/temp_" . self::FILENAME);
|
||||
file_put_contents($tempFilePath, $disk->get($filePath));
|
||||
return $tempFilePath;
|
||||
}
|
||||
|
||||
private function processFile(string $tempFilePath, string $filePath)
|
||||
: void
|
||||
{
|
||||
$handle = fopen($tempFilePath, "r");
|
||||
if ($handle === false) {
|
||||
Log::error("Unable to open file: $filePath");
|
||||
return;
|
||||
}
|
||||
|
||||
$headers = (new TempStmtNarrFormat())->getFillable();
|
||||
$rowCount = 0;
|
||||
|
||||
while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) {
|
||||
$rowCount++;
|
||||
$this->processRow($row, $headers, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors.");
|
||||
}
|
||||
|
||||
private function processRow(array $row, array $headers, int $rowCount, string $filePath)
|
||||
: void
|
||||
{
|
||||
if (count($headers) !== count($row)) {
|
||||
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " .
|
||||
count($headers) . ", Got: " . count($row));
|
||||
return;
|
||||
}
|
||||
|
||||
$data = array_combine($headers, $row);
|
||||
$this->saveRecord($data, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
private function saveRecord(array $data, int $rowCount, string $filePath)
|
||||
: void
|
||||
{
|
||||
try {
|
||||
if (isset($data['_id']) && $data['_id'] !== 'id' && $data['_id'] !== '_id') {
|
||||
TempStmtNarrFormat::updateOrCreate(['_id' => $data['_id']], $data);
|
||||
$this->processedCount++;
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
$this->errorCount++;
|
||||
Log::error("Error processing Statement Narrative Format at row $rowCount in $filePath: " . $e->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private function cleanup(string $tempFilePath)
|
||||
: void
|
||||
{
|
||||
if (file_exists($tempFilePath)) {
|
||||
unlink($tempFilePath);
|
||||
}
|
||||
}
|
||||
|
||||
private function logJobCompletion()
|
||||
: void
|
||||
{
|
||||
Log::info("Statement Narrative Format data processing completed. " .
|
||||
"Total processed: {$this->processedCount}, Total errors: {$this->errorCount}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,98 +16,149 @@
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
protected array $periods;
|
||||
private const CSV_DELIMITER = '~';
|
||||
private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds
|
||||
private const FILENAME = 'ST.STMT.NARR.PARAM.csv';
|
||||
private const DISK_NAME = 'sftpStatement';
|
||||
|
||||
private string $period;
|
||||
private int $processedCount = 0;
|
||||
private int $errorCount = 0;
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
*/
|
||||
public function __construct(array|string $periods = [])
|
||||
public function __construct(string $period = '')
|
||||
{
|
||||
$this->periods = is_string($periods) ? [$periods] : $periods;
|
||||
$this->period = $period;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Execute the job.
|
||||
*/
|
||||
public function handle(): void
|
||||
public function handle()
|
||||
: void
|
||||
{
|
||||
try {
|
||||
set_time_limit(24 * 60 * 60);
|
||||
$disk = Storage::disk('sftpStatement');
|
||||
$processedCount = 0;
|
||||
$errorCount = 0;
|
||||
$this->initializeJob();
|
||||
|
||||
if (empty($this->periods)) {
|
||||
if ($this->period === '') {
|
||||
Log::warning('No periods provided for statement narrative parameter data processing');
|
||||
return;
|
||||
}
|
||||
|
||||
foreach ($this->periods as $period) {
|
||||
// Skip the _parameter folder
|
||||
/*if ($period === '_parameter') {
|
||||
Log::info("Skipping _parameter folder");
|
||||
continue;
|
||||
}*/
|
||||
|
||||
// Construct the filename based on the period folder name
|
||||
$filename = "ST.STMT.NARR.PARAM.csv";
|
||||
$filePath = "$period/$filename";
|
||||
|
||||
Log::info("Processing statement narrative parameter file: $filePath");
|
||||
|
||||
if (!$disk->exists($filePath)) {
|
||||
Log::warning("File not found: $filePath");
|
||||
continue;
|
||||
}
|
||||
|
||||
// Create a temporary local copy of the file
|
||||
$tempFilePath = storage_path("app/temp_$filename");
|
||||
file_put_contents($tempFilePath, $disk->get($filePath));
|
||||
|
||||
$handle = fopen($tempFilePath, "r");
|
||||
|
||||
if ($handle !== false) {
|
||||
$headers = (new TempStmtNarrParam())->getFillable();
|
||||
$rowCount = 0;
|
||||
|
||||
while (($row = fgetcsv($handle, 0, "~")) !== false) {
|
||||
$rowCount++;
|
||||
|
||||
if (count($headers) === count($row)) {
|
||||
$data = array_combine($headers, $row);
|
||||
try {
|
||||
if (isset($data['_id']) && $data['_id'] !== 'id' && $data['_id'] !== '_id') {
|
||||
TempStmtNarrParam::updateOrCreate(
|
||||
['_id' => $data['_id']],
|
||||
$data
|
||||
);
|
||||
$processedCount++;
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
$errorCount++;
|
||||
Log::error("Error processing Statement Narrative Parameter at row $rowCount in $filePath: " . $e->getMessage());
|
||||
}
|
||||
} else {
|
||||
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row));
|
||||
}
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors.");
|
||||
|
||||
// Clean up the temporary file
|
||||
unlink($tempFilePath);
|
||||
} else {
|
||||
Log::error("Unable to open file: $filePath");
|
||||
}
|
||||
}
|
||||
|
||||
Log::info("Statement Narrative Parameter data processing completed. Total processed: $processedCount, Total errors: $errorCount");
|
||||
|
||||
$this->processPeriod();
|
||||
$this->logJobCompletion();
|
||||
} catch (Exception $e) {
|
||||
Log::error('Error in ProcessStmtNarrParamDataJob: ' . $e->getMessage());
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
private function initializeJob()
|
||||
: void
|
||||
{
|
||||
set_time_limit(self::MAX_EXECUTION_TIME);
|
||||
$this->processedCount = 0;
|
||||
$this->errorCount = 0;
|
||||
}
|
||||
|
||||
private function processPeriod()
|
||||
: void
|
||||
{
|
||||
$disk = Storage::disk(self::DISK_NAME);
|
||||
$filePath = "$this->period/" . self::FILENAME;
|
||||
|
||||
if (!$this->validateFile($disk, $filePath)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$tempFilePath = $this->createTemporaryFile($disk, $filePath);
|
||||
$this->processFile($tempFilePath, $filePath);
|
||||
$this->cleanup($tempFilePath);
|
||||
}
|
||||
|
||||
private function validateFile($disk, string $filePath)
|
||||
: bool
|
||||
{
|
||||
Log::info("Processing statement narrative parameter file: $filePath");
|
||||
|
||||
if (!$disk->exists($filePath)) {
|
||||
Log::warning("File not found: $filePath");
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private function createTemporaryFile($disk, string $filePath)
|
||||
: string
|
||||
{
|
||||
$tempFilePath = storage_path("app/temp_" . self::FILENAME);
|
||||
file_put_contents($tempFilePath, $disk->get($filePath));
|
||||
return $tempFilePath;
|
||||
}
|
||||
|
||||
private function processFile(string $tempFilePath, string $filePath)
|
||||
: void
|
||||
{
|
||||
$handle = fopen($tempFilePath, "r");
|
||||
if ($handle === false) {
|
||||
Log::error("Unable to open file: $filePath");
|
||||
return;
|
||||
}
|
||||
|
||||
$headers = (new TempStmtNarrParam())->getFillable();
|
||||
$rowCount = 0;
|
||||
|
||||
while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) {
|
||||
$rowCount++;
|
||||
$this->processRow($row, $headers, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors.");
|
||||
}
|
||||
|
||||
private function processRow(array $row, array $headers, int $rowCount, string $filePath)
|
||||
: void
|
||||
{
|
||||
if (count($headers) !== count($row)) {
|
||||
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " .
|
||||
count($headers) . ", Got: " . count($row));
|
||||
return;
|
||||
}
|
||||
|
||||
$data = array_combine($headers, $row);
|
||||
$this->saveRecord($data, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
private function saveRecord(array $data, int $rowCount, string $filePath)
|
||||
: void
|
||||
{
|
||||
try {
|
||||
if (isset($data['_id']) && $data['_id'] !== 'id' && $data['_id'] !== '_id') {
|
||||
TempStmtNarrParam::updateOrCreate(['_id' => $data['_id']], $data);
|
||||
$this->processedCount++;
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
$this->errorCount++;
|
||||
Log::error("Error processing Statement Narrative Parameter at row $rowCount in $filePath: " . $e->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private function cleanup(string $tempFilePath)
|
||||
: void
|
||||
{
|
||||
if (file_exists($tempFilePath)) {
|
||||
unlink($tempFilePath);
|
||||
}
|
||||
}
|
||||
|
||||
private function logJobCompletion()
|
||||
: void
|
||||
{
|
||||
Log::info("Statement Narrative Parameter data processing completed. " .
|
||||
"Total processed: {$this->processedCount}, Total errors: {$this->errorCount}");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,11 +16,11 @@
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
// Konstanta untuk nilai-nilai statis
|
||||
private const FILE_EXTENSION = '.ST.TELLER.csv';
|
||||
private const CSV_DELIMITER = '~';
|
||||
private const DISK_NAME = 'sftpStatement';
|
||||
private const HEADER_MAP = [
|
||||
private const CSV_DELIMITER = '~';
|
||||
private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds
|
||||
private const FILENAME = 'ST.TELLER.csv';
|
||||
private const DISK_NAME = 'sftpStatement';
|
||||
private const HEADER_MAP = [
|
||||
'id' => 'id_teller',
|
||||
'account_1' => 'account_1',
|
||||
'currency_1' => 'currency_1',
|
||||
@@ -126,8 +126,9 @@
|
||||
'last_version' => 'last_version'
|
||||
];
|
||||
|
||||
// Pemetaan bidang header ke kolom model
|
||||
protected string $period;
|
||||
private string $period;
|
||||
private int $processedCount = 0;
|
||||
private int $errorCount = 0;
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
@@ -144,120 +145,94 @@
|
||||
: void
|
||||
{
|
||||
try {
|
||||
set_time_limit(24 * 60 * 60);
|
||||
$this->initializeJob();
|
||||
|
||||
if ($this->period === '') {
|
||||
Log::warning('No period provided for teller data processing');
|
||||
return;
|
||||
}
|
||||
|
||||
$stats = $this->processPeriodFile();
|
||||
|
||||
Log::info("ProcessTellerDataJob completed. Total processed: {$stats['processed']}, Total errors: {$stats['errors']}");
|
||||
$this->processPeriod();
|
||||
$this->logJobCompletion();
|
||||
} catch (Exception $e) {
|
||||
Log::error("Error in ProcessTellerDataJob: " . $e->getMessage());
|
||||
Log::error('Error in ProcessTellerDataJob: ' . $e->getMessage());
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a single period file
|
||||
*/
|
||||
private function processPeriodFile()
|
||||
: array
|
||||
private function initializeJob()
|
||||
: void
|
||||
{
|
||||
$disk = Storage::disk(self::DISK_NAME);
|
||||
$filename = $this->period . self::FILE_EXTENSION;
|
||||
$filePath = "{$this->period}/$filename";
|
||||
$processedCount = 0;
|
||||
$errorCount = 0;
|
||||
set_time_limit(self::MAX_EXECUTION_TIME);
|
||||
$this->processedCount = 0;
|
||||
$this->errorCount = 0;
|
||||
}
|
||||
|
||||
private function processPeriod()
|
||||
: void
|
||||
{
|
||||
$disk = Storage::disk(self::DISK_NAME);
|
||||
$filename = "{$this->period}." . self::FILENAME;
|
||||
$filePath = "{$this->period}/$filename";
|
||||
|
||||
if (!$this->validateFile($disk, $filePath)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$tempFilePath = $this->createTemporaryFile($disk, $filePath, $filename);
|
||||
$this->processFile($tempFilePath, $filePath);
|
||||
$this->cleanup($tempFilePath);
|
||||
}
|
||||
|
||||
private function validateFile($disk, string $filePath)
|
||||
: bool
|
||||
{
|
||||
Log::info("Processing teller file: $filePath");
|
||||
|
||||
if (!$disk->exists($filePath)) {
|
||||
Log::warning("File not found: $filePath");
|
||||
return ['processed' => 0, 'errors' => 0];
|
||||
return false;
|
||||
}
|
||||
|
||||
$tempFilePath = $this->createTempFile($disk, $filePath, $filename);
|
||||
|
||||
$result = $this->processCSVFile($tempFilePath, $filePath);
|
||||
$processedCount += $result['processed'];
|
||||
$errorCount += $result['errors'];
|
||||
|
||||
// Clean up the temporary file
|
||||
if (file_exists($tempFilePath)) {
|
||||
unlink($tempFilePath);
|
||||
}
|
||||
|
||||
Log::info("Completed processing $filePath. Processed {$result['processed']} records with {$result['errors']} errors.");
|
||||
|
||||
return [
|
||||
'processed' => $processedCount,
|
||||
'errors' => $errorCount
|
||||
];
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a temporary file for processing
|
||||
*/
|
||||
private function createTempFile($disk, string $filePath, string $filename)
|
||||
private function createTemporaryFile($disk, string $filePath, string $fileName)
|
||||
: string
|
||||
{
|
||||
$tempFilePath = storage_path("app/temp_$filename");
|
||||
$tempFilePath = storage_path("app/temp_$fileName");
|
||||
file_put_contents($tempFilePath, $disk->get($filePath));
|
||||
return $tempFilePath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a CSV file and import data
|
||||
*/
|
||||
private function processCSVFile(string $tempFilePath, string $originalFilePath)
|
||||
: array
|
||||
private function processFile(string $tempFilePath, string $filePath)
|
||||
: void
|
||||
{
|
||||
$processedCount = 0;
|
||||
$errorCount = 0;
|
||||
|
||||
$handle = fopen($tempFilePath, "r");
|
||||
if ($handle === false) {
|
||||
Log::error("Unable to open file: $originalFilePath");
|
||||
return ['processed' => 0, 'errors' => 0];
|
||||
Log::error("Unable to open file: $filePath");
|
||||
return;
|
||||
}
|
||||
|
||||
// Get the headers from the first row
|
||||
$headerRow = fgetcsv($handle, 0, self::CSV_DELIMITER);
|
||||
if (!$headerRow) {
|
||||
fclose($handle);
|
||||
return ['processed' => 0, 'errors' => 0];
|
||||
return;
|
||||
}
|
||||
|
||||
$rowCount = 0;
|
||||
while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) {
|
||||
$rowCount++;
|
||||
|
||||
if (count($headerRow) !== count($row)) {
|
||||
Log::warning("Row $rowCount in $originalFilePath has incorrect column count. Expected: " . count($headerRow) . ", Got: " . count($row));
|
||||
continue;
|
||||
}
|
||||
|
||||
$result = $this->processRow($headerRow, $row, $rowCount, $originalFilePath);
|
||||
$processedCount += $result['processed'];
|
||||
$errorCount += $result['errors'];
|
||||
$this->processRow($headerRow, $row, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
|
||||
return [
|
||||
'processed' => $processedCount,
|
||||
'errors' => $errorCount
|
||||
];
|
||||
Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors.");
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a single row from the CSV file
|
||||
*/
|
||||
private function processRow(array $headerRow, array $row, int $rowCount, string $filePath)
|
||||
: array
|
||||
: void
|
||||
{
|
||||
// Combine the header row with the data row
|
||||
$rawData = array_combine($headerRow, $row);
|
||||
@@ -270,7 +245,7 @@
|
||||
|
||||
// Skip header row if it was included in the data
|
||||
if ($data['id_teller'] === 'id') {
|
||||
return ['processed' => 0, 'errors' => 0];
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
@@ -278,10 +253,24 @@
|
||||
$teller->fill($data);
|
||||
$teller->save();
|
||||
|
||||
return ['processed' => 1, 'errors' => 0];
|
||||
$this->processedCount++;
|
||||
} catch (Exception $e) {
|
||||
Log::error("Error processing Teller at row $rowCount in $filePath: " . $e->getMessage());
|
||||
return ['processed' => 0, 'errors' => 1];
|
||||
$this->errorCount++;
|
||||
}
|
||||
}
|
||||
|
||||
private function logJobCompletion()
|
||||
: void
|
||||
{
|
||||
Log::info("ProcessTellerDataJob completed. Total processed: {$this->processedCount}, Total errors: {$this->errorCount}");
|
||||
}
|
||||
|
||||
private function cleanup(string $tempFilePath)
|
||||
: void
|
||||
{
|
||||
if (file_exists($tempFilePath)) {
|
||||
unlink($tempFilePath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,113 +1,164 @@
|
||||
<?php
|
||||
|
||||
namespace Modules\Webstatement\Jobs;
|
||||
namespace Modules\Webstatement\Jobs;
|
||||
|
||||
use Exception;
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use Modules\Webstatement\Models\TempTransaction;
|
||||
use Exception;
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use Modules\Webstatement\Models\TempTransaction;
|
||||
|
||||
class ProcessTransactionDataJob implements ShouldQueue
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
protected array $periods;
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
*/
|
||||
public function __construct(array|string $periods = [])
|
||||
class ProcessTransactionDataJob implements ShouldQueue
|
||||
{
|
||||
$this->periods = is_string($periods) ? [$periods] : $periods;
|
||||
}
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
private const CSV_DELIMITER = '~';
|
||||
private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds
|
||||
private const FILENAME = 'ST.TRANSACTION.csv';
|
||||
private const DISK_NAME = 'sftpStatement';
|
||||
|
||||
/**
|
||||
* Execute the job.
|
||||
*/
|
||||
public function handle(): void
|
||||
{
|
||||
try {
|
||||
set_time_limit(24 * 60 * 60);
|
||||
$disk = Storage::disk('sftpStatement');
|
||||
$processedCount = 0;
|
||||
$errorCount = 0;
|
||||
private string $period;
|
||||
private int $processedCount = 0;
|
||||
private int $errorCount = 0;
|
||||
|
||||
if (empty($this->periods)) {
|
||||
Log::warning('No periods provided for transaction data processing');
|
||||
/**
|
||||
* Create a new job instance.
|
||||
*/
|
||||
public function __construct(string $period = '')
|
||||
{
|
||||
$this->period = $period;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the job.
|
||||
*/
|
||||
public function handle()
|
||||
: void
|
||||
{
|
||||
try {
|
||||
$this->initializeJob();
|
||||
|
||||
if ($this->period === '') {
|
||||
Log::warning('No periods provided for transaction data processing');
|
||||
return;
|
||||
}
|
||||
|
||||
$this->processPeriod();
|
||||
$this->logJobCompletion();
|
||||
} catch (Exception $e) {
|
||||
Log::error('Error in ProcessTransactionDataJob: ' . $e->getMessage());
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
private function initializeJob()
|
||||
: void
|
||||
{
|
||||
set_time_limit(self::MAX_EXECUTION_TIME);
|
||||
$this->processedCount = 0;
|
||||
$this->errorCount = 0;
|
||||
}
|
||||
|
||||
private function processPeriod()
|
||||
: void
|
||||
{
|
||||
$disk = Storage::disk(self::DISK_NAME);
|
||||
$filePath = "$this->period/" . self::FILENAME;
|
||||
|
||||
if (!$this->validateFile($disk, $filePath)) {
|
||||
return;
|
||||
}
|
||||
|
||||
foreach ($this->periods as $period) {
|
||||
// Skip the _parameter folder
|
||||
/*if ($period === '_parameter') {
|
||||
Log::info("Skipping _parameter folder");
|
||||
continue;
|
||||
}*/
|
||||
$tempFilePath = $this->createTemporaryFile($disk, $filePath);
|
||||
$this->processFile($tempFilePath, $filePath);
|
||||
$this->cleanup($tempFilePath);
|
||||
}
|
||||
|
||||
// Construct the filename based on the period folder name
|
||||
$filename = "ST.TRANSACTION.csv";
|
||||
$filePath = "$period/$filename";
|
||||
private function validateFile($disk, string $filePath)
|
||||
: bool
|
||||
{
|
||||
Log::info("Processing transaction file: $filePath");
|
||||
|
||||
Log::info("Processing transaction file: $filePath");
|
||||
|
||||
if (!$disk->exists($filePath)) {
|
||||
Log::warning("File not found: $filePath");
|
||||
continue;
|
||||
}
|
||||
|
||||
// Create a temporary local copy of the file
|
||||
$tempFilePath = storage_path("app/temp_$filename");
|
||||
file_put_contents($tempFilePath, $disk->get($filePath));
|
||||
|
||||
$handle = fopen($tempFilePath, "r");
|
||||
|
||||
if ($handle !== false) {
|
||||
$headers = (new TempTransaction())->getFillable();
|
||||
$rowCount = 0;
|
||||
|
||||
while (($row = fgetcsv($handle, 0, "~")) !== false) {
|
||||
$rowCount++;
|
||||
|
||||
if (count($headers) === count($row)) {
|
||||
$data = array_combine($headers, $row);
|
||||
try {
|
||||
if (isset($data['_id']) && $data['_id'] !== 'id' && $data['_id'] !== '_id') {
|
||||
TempTransaction::updateOrCreate(
|
||||
['_id' => $data['_id']],
|
||||
$data
|
||||
);
|
||||
$processedCount++;
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
$errorCount++;
|
||||
Log::error("Error processing Transaction at row $rowCount in $filePath: " . $e->getMessage());
|
||||
}
|
||||
} else {
|
||||
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row));
|
||||
}
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors.");
|
||||
|
||||
// Clean up the temporary file
|
||||
unlink($tempFilePath);
|
||||
} else {
|
||||
Log::error("Unable to open file: $filePath");
|
||||
}
|
||||
if (!$disk->exists($filePath)) {
|
||||
Log::warning("File not found: $filePath");
|
||||
return false;
|
||||
}
|
||||
|
||||
Log::info("Transaction data processing completed. Total processed: $processedCount, Total errors: $errorCount");
|
||||
return true;
|
||||
}
|
||||
|
||||
} catch (Exception $e) {
|
||||
Log::error('Error in ProcessTransactionDataJob: ' . $e->getMessage());
|
||||
throw $e;
|
||||
private function createTemporaryFile($disk, string $filePath)
|
||||
: string
|
||||
{
|
||||
$tempFilePath = storage_path("app/temp_" . self::FILENAME);
|
||||
file_put_contents($tempFilePath, $disk->get($filePath));
|
||||
return $tempFilePath;
|
||||
}
|
||||
|
||||
private function processFile(string $tempFilePath, string $filePath)
|
||||
: void
|
||||
{
|
||||
$handle = fopen($tempFilePath, "r");
|
||||
if ($handle === false) {
|
||||
Log::error("Unable to open file: $filePath");
|
||||
return;
|
||||
}
|
||||
|
||||
$headers = (new TempTransaction())->getFillable();
|
||||
$rowCount = 0;
|
||||
|
||||
while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) {
|
||||
$rowCount++;
|
||||
$this->processRow($row, $headers, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors.");
|
||||
}
|
||||
|
||||
private function processRow(array $row, array $headers, int $rowCount, string $filePath)
|
||||
: void
|
||||
{
|
||||
if (count($headers) !== count($row)) {
|
||||
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " .
|
||||
count($headers) . ", Got: " . count($row));
|
||||
return;
|
||||
}
|
||||
|
||||
$data = array_combine($headers, $row);
|
||||
$this->saveRecord($data, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
private function saveRecord(array $data, int $rowCount, string $filePath)
|
||||
: void
|
||||
{
|
||||
try {
|
||||
if (isset($data['_id']) && $data['_id'] !== 'id' && $data['_id'] !== '_id') {
|
||||
TempTransaction::updateOrCreate(['_id' => $data['_id']], $data);
|
||||
$this->processedCount++;
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
$this->errorCount++;
|
||||
Log::error("Error processing Transaction at row $rowCount in $filePath: " . $e->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private function cleanup(string $tempFilePath)
|
||||
: void
|
||||
{
|
||||
if (file_exists($tempFilePath)) {
|
||||
unlink($tempFilePath);
|
||||
}
|
||||
}
|
||||
|
||||
private function logJobCompletion()
|
||||
: void
|
||||
{
|
||||
Log::info("Transaction data processing completed. " .
|
||||
"Total processed: {$this->processedCount}, Total errors: {$this->errorCount}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user