refactor(webstatement): sederhanakan logika pemrosesan data transfer dana

- Menghapus penggunaan batching data (`transferBatch`) untuk penyederhanaan logika:
  - Menghapus konstanta `CHUNK_SIZE`.
  - Menghapus variabel dan metode terkait batching, seperti `transferBatch`, `addToBatch`, dan `saveBatch`.
  - Memproses dan menyimpan setiap baris data langsung melalui metode `saveRecord`.

- Memperbarui metode `processRow`:
  - Menggantikan logika penambahan batch dengan langsung memanggil `saveRecord`.

Signed-off-by: Daeng Deni Mardaeni <ddeni05@gmail.com>
This commit is contained in:
Daeng Deni Mardaeni
2025-06-06 21:00:12 +07:00
parent 47bf23302f
commit 58a5db7303

View File

@@ -20,12 +20,10 @@
private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds
private const FILENAME = 'ST.FUNDS.TRANSFER.csv'; private const FILENAME = 'ST.FUNDS.TRANSFER.csv';
private const DISK_NAME = 'sftpStatement'; private const DISK_NAME = 'sftpStatement';
private const CHUNK_SIZE = 1000; // Process data in chunks to reduce memory usage
private string $period = ''; private string $period = '';
private int $processedCount = 0; private int $processedCount = 0;
private int $errorCount = 0; private int $errorCount = 0;
private array $transferBatch = [];
/** /**
* Create a new job instance. * Create a new job instance.
@@ -63,7 +61,6 @@
set_time_limit(self::MAX_EXECUTION_TIME); set_time_limit(self::MAX_EXECUTION_TIME);
$this->processedCount = 0; $this->processedCount = 0;
$this->errorCount = 0; $this->errorCount = 0;
$this->transferBatch = [];
} }
private function processPeriod() private function processPeriod()
@@ -114,23 +111,10 @@
$headers = (new TempFundsTransfer())->getFillable(); $headers = (new TempFundsTransfer())->getFillable();
$rowCount = 0; $rowCount = 0;
$chunkCount = 0;
while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) { while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) {
$rowCount++; $rowCount++;
$this->processRow($row, $headers, $rowCount, $filePath); $this->processRow($row, $headers, $rowCount, $filePath);
// Process in chunks to avoid memory issues
if (count($this->transferBatch) >= self::CHUNK_SIZE) {
$this->saveBatch();
$chunkCount++;
Log::info("Processed chunk $chunkCount ({$this->processedCount} records so far)");
}
}
// Process any remaining records
if (!empty($this->transferBatch)) {
$this->saveBatch();
} }
fclose($handle); fclose($handle);
@@ -152,24 +136,18 @@
} }
$data = array_combine($headers, $row); $data = array_combine($headers, $row);
$this->addToBatch($data, $rowCount, $filePath); $this->saveRecord($data, $rowCount, $filePath);
} }
/** private function saveRecord(array $data, int $rowCount, string $filePath)
* Add record to batch instead of saving immediately
*/
private function addToBatch(array $data, int $rowCount, string $filePath)
: void : void
{ {
try { try {
if (isset($data['_id']) && $data['_id'] !== '_id') { if (isset($data['_id']) && $data['_id'] !== '_id') {
// Add timestamp fields TempFundsTransfer::updateOrCreate(
$now = now(); ['_id' => $data['_id']],
$data['created_at'] = $now; $data
$data['updated_at'] = $now; );
// Add to transfer batch
$this->transferBatch[] = $data;
$this->processedCount++; $this->processedCount++;
} }
} catch (Exception $e) { } catch (Exception $e) {
@@ -178,37 +156,6 @@
} }
} }
/**
* Save batched records to the database
*/
private function saveBatch()
: void
{
try {
if (!empty($this->transferBatch)) {
// Process in smaller chunks for better memory management
foreach ($this->transferBatch as $entry) {
// Extract all stmt_entry_ids from the current chunk
$entryIds = array_column($entry, '_id');
// Delete existing records with these IDs to avoid conflicts
TempFundsTransfer::whereIn('_id', $entryIds)->delete();
// Insert all records in the chunk at once
TempFundsTransfer::insert($entry);
}
// Reset entry batch after processing
$this->transferBatch = [];
}
} catch (Exception $e) {
Log::error("Error in saveBatch: " . $e->getMessage());
$this->errorCount += count($this->transferBatch);
// Reset batch even if there's an error to prevent reprocessing the same failed records
$this->transferBatch = [];
}
}
private function cleanup(string $tempFilePath) private function cleanup(string $tempFilePath)
: void : void
{ {