daengdeni/webstatement
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(jobs): optimize job classes by adding modular methods and constants

Browse Source 
- Menambahkan konstanta baru pada setiap job untuk meningkatkan keterbacaan:
  - `CSV_DELIMITER` untuk delimiter CSV.
  - `MAX_EXECUTION_TIME` untuk batas waktu eksekusi (86400 detik).
  - `FILENAME` untuk nama file masing-masing job.
  - `DISK_NAME` untuk disk yang digunakan.
- Mengubah `protected` menjadi `private` untuk properti seperti:
  - `$period`, `$processedCount`, dan `$errorCount` di semua job.
- Memindahkan logika proses menjadi metode modular untuk meningkatkan modularitas:
  - Metode `initializeJob` untuk inisialisasi counter.
  - Metode `processPeriod` untuk menangani file dan memulai proses.
  - Metode `validateFile` untuk validasi keberadaan file.
  - Metode `createTemporaryFile` untuk menyalin file sementara.
  - Metode `processFile` untuk membaca isi file CSV dan memproses.
  - Metode `processRow`/`mapAndSaveRecord`/`saveRecord` untuk pemrosesan dan penyimpanan data.
  - Metode `cleanup` untuk menghapus file sementara.
  - Metode `logJobCompletion` untuk logging hasil akhir.
- Menerapkan pengolahan file dengan sistem logging terperinci:
  - Logging row dengan kolom tidak sesuai akan menghasilkan peringatan.
  - Mencatat jumlah record berhasil diproses serta error.

Refaktor ini bertujuan untuk meningkatkan kualitas kode melalui modularisasi, keterbacaan, dan kemudahan pengujian ulang di seluruh kelas job.
This commit is contained in:
daengdeni
2025-05-26 15:26:43 +07:00
parent 41ed7c1ed9
commit c6363473ac
15 changed files with 2035 additions and 1250 deletions
Download Patch File Download Diff File Expand all files Collapse all files

330
app/Jobs/ProcessDataCaptureDataJob.php
View File

@@ -16,16 +16,63 @@
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $period;
protected $filename;
private const CSV_DELIMITER = '~';
private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds
private const FILENAME = 'ST.DATA.CAPTURE.csv';
private const DISK_NAME = 'sftpStatement';
private const CSV_HEADERS = [
'id',
'account_number',
'sign',
'amount_lcy',
'transaction_code',
'their_reference',
'narrative',
'pl_category',
'customer_id',
'account_officer',
'product_category',
'value_date',
'currency',
'amount_fcy',
'exchange_rate',
'neg_ref_no',
'position_type',
'our_reference',
'reversal_marker',
'exposure_date',
'currency_market',
'iblc_country',
'last_version',
'otor_version',
'department_code',
'dealer_desk',
'bank_sort_cde',
'cheque_number',
'accounting_date',
'contingent_acct',
'cheq_type',
'tfs_reference',
'accounting_company',
'stmt_no',
'curr_no',
'inputter',
'authoriser',
'co_code',
'date_time'
];
private const DATE_FIELDS = ['value_date', 'exposure_date', 'accounting_date'];
private string $period;
private int $processedCount = 0;
private int $errorCount = 0;
/**
* Create a new job instance.
*/
public function __construct(string $period = '', string $filename = "ST.DATA.CAPTURE.csv")
public function __construct(string $period = '')
{
$this->period = $period;
$this->filename = $filename;
$this->period = $period;
}
/**
@@ -35,143 +82,160 @@
: void
{
try {
set_time_limit(24 * 60 * 60);
$disk = Storage::disk('sftpStatement');
$processedCount = 0;
$errorCount = 0;
$this->initializeJob();
if ($this->period === '') {
Log::warning('No period provided for data capture processing');
return;
}
// Construct the filepath based on the period folder name
$fileName = "{$this->period}.{$this->filename}";
$filePath = "{$this->period}/$fileName";
Log::info("Processing data capture file: $filePath");
if (!$disk->exists($filePath)) {
Log::warning("File not found: $filePath");
return;
}
// Create a temporary local copy of the file
$tempFilePath = storage_path("app/temp_{$this->filename}");
file_put_contents($tempFilePath, $disk->get($filePath));
$handle = fopen($tempFilePath, "r");
if ($handle !== false) {
// CSV headers from the file
$csvHeaders = [
'id',
'account_number',
'sign',
'amount_lcy',
'transaction_code',
'their_reference',
'narrative',
'pl_category',
'customer_id',
'account_officer',
'product_category',
'value_date',
'currency',
'amount_fcy',
'exchange_rate',
'neg_ref_no',
'position_type',
'our_reference',
'reversal_marker',
'exposure_date',
'currency_market',
'iblc_country',
'last_version',
'otor_version',
'department_code',
'dealer_desk',
'bank_sort_cde',
'cheque_number',
'accounting_date',
'contingent_acct',
'cheq_type',
'tfs_reference',
'accounting_company',
'stmt_no',
'curr_no',
'inputter',
'authoriser',
'co_code',
'date_time'
];
$rowCount = 0;
while (($row = fgetcsv($handle, 0, "~")) !== false) {
$rowCount++;
// Skip header row if it exists
if ($rowCount === 1 && strtolower($row[0]) === 'id') {
continue;
}
if (count($csvHeaders) === count($row)) {
$data = array_combine($csvHeaders, $row);
try {
// Format dates if they exist
foreach (['value_date', 'exposure_date', 'accounting_date'] as $dateField) {
if (!empty($data[$dateField])) {
try {
$data[$dateField] = date('Y-m-d', strtotime($data[$dateField]));
} catch (Exception $e) {
// If date parsing fails, keep the original value
Log::warning("Failed to parse date for $dateField: {$data[$dateField]}");
}
}
}
// Format datetime if it exists
if (!empty($data['date_time'])) {
try {
$data['date_time'] = date('Y-m-d H:i:s', strtotime($data['date_time']));
} catch (Exception $e) {
// If datetime parsing fails, keep the original value
Log::warning("Failed to parse datetime for date_time: {$data['date_time']}");
}
}
if (!empty($data['id'])) {
DataCapture::updateOrCreate(
['id' => $data['id']],
$data
);
$processedCount++;
}
} catch (Exception $e) {
$errorCount++;
Log::error("Error processing Data Capture at row $rowCount in $filePath: " . $e->getMessage());
}
} else {
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($csvHeaders) . ", Got: " . count($row));
}
}
fclose($handle);
Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors.");
// Clean up the temporary file
unlink($tempFilePath);
} else {
Log::error("Unable to open file: $filePath");
}
Log::info("Data capture processing completed. Total processed: $processedCount, Total errors: $errorCount");
$this->processPeriod();
$this->logJobCompletion();
} catch (Exception $e) {
Log::error('Error in ProcessDataCaptureDataJob: ' . $e->getMessage());
throw $e;
}
}
private function initializeJob()
: void
{
set_time_limit(self::MAX_EXECUTION_TIME);
$this->processedCount = 0;
$this->errorCount = 0;
}
private function processPeriod()
: void
{
$disk = Storage::disk(self::DISK_NAME);
$filename = "{$this->period}." . self::FILENAME;
$filePath = "{$this->period}/$filename";
if (!$this->validateFile($disk, $filePath)) {
return;
}
$tempFilePath = $this->createTemporaryFile($disk, $filePath, $filename);
$this->processFile($tempFilePath, $filePath);
$this->cleanup($tempFilePath);
}
private function validateFile($disk, string $filePath)
: bool
{
Log::info("Processing data capture file: $filePath");
if (!$disk->exists($filePath)) {
Log::warning("File not found: $filePath");
return false;
}
return true;
}
private function createTemporaryFile($disk, string $filePath, string $filename)
: string
{
$tempFilePath = storage_path("app/temp_$filename");
file_put_contents($tempFilePath, $disk->get($filePath));
return $tempFilePath;
}
private function processFile(string $tempFilePath, string $filePath)
: void
{
$handle = fopen($tempFilePath, "r");
if ($handle === false) {
Log::error("Unable to open file: $filePath");
return;
}
$rowCount = 0;
while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) {
$rowCount++;
// Skip header row if it exists
if ($rowCount === 1 && strtolower($row[0]) === 'id') {
continue;
}
$this->processRow($row, $rowCount, $filePath);
}
fclose($handle);
Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors.");
}
private function processRow(array $row, int $rowCount, string $filePath)
: void
{
if (count(self::CSV_HEADERS) !== count($row)) {
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " .
count(self::CSV_HEADERS) . ", Got: " . count($row));
return;
}
$data = array_combine(self::CSV_HEADERS, $row);
$this->formatDates($data);
$this->saveRecord($data, $rowCount, $filePath);
}
private function formatDates(array &$data)
: void
{
// Format dates if they exist
foreach (self::DATE_FIELDS as $dateField) {
if (!empty($data[$dateField])) {
try {
$data[$dateField] = date('Y-m-d', strtotime($data[$dateField]));
} catch (Exception $e) {
// If date parsing fails, keep the original value
Log::warning("Failed to parse date for $dateField: {$data[$dateField]}");
}
}
}
// Format datetime if it exists
if (!empty($data['date_time'])) {
try {
$data['date_time'] = date('Y-m-d H:i:s', strtotime($data['date_time']));
} catch (Exception $e) {
// If datetime parsing fails, keep the original value
Log::warning("Failed to parse datetime for date_time: {$data['date_time']}");
}
}
}
private function saveRecord(array $data, int $rowCount, string $filePath)
: void
{
try {
if (!empty($data['id'])) {
DataCapture::updateOrCreate(
['id' => $data['id']],
$data
);
$this->processedCount++;
}
} catch (Exception $e) {
$this->errorCount++;
Log::error("Error processing Data Capture at row $rowCount in $filePath: " . $e->getMessage());
}
}
private function cleanup(string $tempFilePath)
: void
{
if (file_exists($tempFilePath)) {
unlink($tempFilePath);
}
}
private function logJobCompletion()
: void
{
Log::info("Data capture processing completed. " .
"Total processed: {$this->processedCount}, Total errors: {$this->errorCount}");
}
}