feat(webstatement): implement batch processing in job data handling
- Menambahkan konstanta `CHUNK_SIZE` dengan nilai 1000 untuk memungkinkan pemrosesan data dalam bentuk batch guna mengurangi penggunaan memori. - Memperkenalkan atribut batch baru (`atmTransactionBatch`, `captureBatch`, `transferBatch`, dan `tellerBatch`) untuk menyimpan data sementara sebelum disimpan ke database. - Mengganti metode penyimpanan langsung dengan menambahkan data ke batch melalui fungsi baru `addToBatch()`. - Menambahkan fungsi `saveBatch()` untuk melakukan operasi penyimpanan data dalam jumlah besar (bulk) menggunakan metode `upsert`. - Memastikan batch akan di-reset setelah penyimpanan, termasuk dalam kasus kegagalan, untuk mencegah reprocessing catatan yang gagal. - Menambahkan logging tambahan untuk mencatat jumlah batch (chunk) yang telah selesai diproses, memberikan wawasan lebih dalam proses. - Menambahkan validasi jumlah kolom pada setiap baris data, logging peringatan jika data tidak sesuai, dan mencatat jumlah kesalahan (`errorCount`). - Secara otomatis menambahkan atribut timestamp (`created_at` dan `updated_at`) pada setiap data sebelum dimasukkan ke dalam batch untuk pelacakan waktu. - Memodifikasi log error untuk menangani kesalahan pada level yang lebih spesifik, seperti pada baris tertentu dalam file yang diproses. - Mengoptimalkan pemrosesan data pada job: - `ProcessAtmTransactionJob` untuk data transaksi ATM. - `ProcessDataCaptureDataJob` untuk data capture. - `ProcessFundsTransferDataJob` untuk data transfer dana. - `ProcessTellerDataJob` untuk data teller. - Meningkatkan efisiensi dan skalabilitas dengan menyisipkan data secara bulk, mengurangi overhead database, dan menghindari pengolahan data secara satu per satu.
This commit is contained in:
daengdeni
@@ -20,6 +20,7 @@
|
||||
private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds
|
||||
private const FILENAME = 'ST.ATM.TRANSACTION.csv';
|
||||
private const DISK_NAME = 'sftpStatement';
|
||||
private const CHUNK_SIZE = 1000; // Process data in chunks to reduce memory usage
|
||||
private const HEADER_MAP = [
|
||||
'id' => 'transaction_id',
|
||||
'card_acc_id' => 'card_acc_id',
|
||||
@@ -43,6 +44,7 @@
|
||||
private string $period = '';
|
||||
private int $processedCount = 0;
|
||||
private int $errorCount = 0;
|
||||
private array $atmTransactionBatch = [];
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
@@ -80,6 +82,7 @@
|
||||
set_time_limit(self::MAX_EXECUTION_TIME);
|
||||
$this->processedCount = 0;
|
||||
$this->errorCount = 0;
|
||||
$this->atmTransactionBatch = [];
|
||||
}
|
||||
|
||||
private function processPeriod()
|
||||
@@ -136,9 +139,23 @@
|
||||
}
|
||||
|
||||
$rowCount = 0;
|
||||
$chunkCount = 0;
|
||||
|
||||
while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) {
|
||||
$rowCount++;
|
||||
$this->processRow($headerRow, $row, $rowCount, $filePath);
|
||||
|
||||
// Process in chunks to avoid memory issues
|
||||
if (count($this->atmTransactionBatch) >= self::CHUNK_SIZE) {
|
||||
$this->saveBatch();
|
||||
$chunkCount++;
|
||||
Log::info("Processed chunk $chunkCount ({$this->processedCount} records so far)");
|
||||
}
|
||||
}
|
||||
|
||||
// Process any remaining records
|
||||
if (!empty($this->atmTransactionBatch)) {
|
||||
$this->saveBatch();
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
@@ -151,15 +168,16 @@
|
||||
if (count($headerRow) !== count($row)) {
|
||||
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " .
|
||||
count($headerRow) . ", Got: " . count($row));
|
||||
$this->errorCount++;
|
||||
return;
|
||||
}
|
||||
|
||||
// Combine the header row with the data row
|
||||
$rawData = array_combine($headerRow, $row);
|
||||
$this->mapAndSaveRecord($rawData, $rowCount, $filePath);
|
||||
$this->mapAndAddToBatch($rawData, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
private function mapAndSaveRecord(array $rawData, int $rowCount, string $filePath)
|
||||
private function mapAndAddToBatch(array $rawData, int $rowCount, string $filePath)
|
||||
: void
|
||||
{
|
||||
// Map the raw data to our model fields
|
||||
@@ -173,24 +191,53 @@
|
||||
return;
|
||||
}
|
||||
|
||||
$this->saveRecord($data, $rowCount, $filePath);
|
||||
$this->addToBatch($data, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
private function saveRecord(array $data, int $rowCount, string $filePath)
|
||||
/**
|
||||
* Add record to batch instead of saving immediately
|
||||
*/
|
||||
private function addToBatch(array $data, int $rowCount, string $filePath)
|
||||
: void
|
||||
{
|
||||
try {
|
||||
// Add timestamp fields
|
||||
$now = now();
|
||||
$data['created_at'] = $now;
|
||||
$data['updated_at'] = $now;
|
||||
|
||||
// Create or update the record
|
||||
AtmTransaction::updateOrCreate(
|
||||
['transaction_id' => $data['transaction_id']],
|
||||
$data
|
||||
);
|
||||
|
||||
// Add to batch
|
||||
$this->atmTransactionBatch[] = $data;
|
||||
$this->processedCount++;
|
||||
} catch (Exception $e) {
|
||||
$this->errorCount++;
|
||||
Log::error("Error processing row $rowCount in $filePath: " . $e->getMessage());
|
||||
Log::error("Error processing ATM Transaction at row $rowCount in $filePath: " . $e->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save batched records to the database
|
||||
*/
|
||||
private function saveBatch()
|
||||
: void
|
||||
{
|
||||
try {
|
||||
if (!empty($this->atmTransactionBatch)) {
|
||||
// Bulk insert/update ATM transactions
|
||||
AtmTransaction::upsert(
|
||||
$this->atmTransactionBatch,
|
||||
['transaction_id'], // Unique key
|
||||
array_values(array_diff(self::HEADER_MAP, ['id'])) // Update columns (all except transaction_id)
|
||||
);
|
||||
|
||||
// Reset batch after processing
|
||||
$this->atmTransactionBatch = [];
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
Log::error("Error in saveBatch: " . $e->getMessage());
|
||||
$this->errorCount += count($this->atmTransactionBatch);
|
||||
// Reset batch even if there's an error to prevent reprocessing the same failed records
|
||||
$this->atmTransactionBatch = [];
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user