refactor(webstatement): sederhanakan logika pemrosesan data transfer dana
- Menghapus penggunaan batching data (`transferBatch`) untuk penyederhanaan logika: - Menghapus konstanta `CHUNK_SIZE`. - Menghapus variabel dan metode terkait batching, seperti `transferBatch`, `addToBatch`, dan `saveBatch`. - Memproses dan menyimpan setiap baris data langsung melalui metode `saveRecord`. - Memperbarui metode `processRow`: - Menggantikan logika penambahan batch dengan langsung memanggil `saveRecord`. Signed-off-by: Daeng Deni Mardaeni <ddeni05@gmail.com>
This commit is contained in:
@@ -20,12 +20,10 @@
|
|||||||
private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds
|
private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds
|
||||||
private const FILENAME = 'ST.FUNDS.TRANSFER.csv';
|
private const FILENAME = 'ST.FUNDS.TRANSFER.csv';
|
||||||
private const DISK_NAME = 'sftpStatement';
|
private const DISK_NAME = 'sftpStatement';
|
||||||
private const CHUNK_SIZE = 1000; // Process data in chunks to reduce memory usage
|
|
||||||
|
|
||||||
private string $period = '';
|
private string $period = '';
|
||||||
private int $processedCount = 0;
|
private int $processedCount = 0;
|
||||||
private int $errorCount = 0;
|
private int $errorCount = 0;
|
||||||
private array $transferBatch = [];
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Create a new job instance.
|
* Create a new job instance.
|
||||||
@@ -63,7 +61,6 @@
|
|||||||
set_time_limit(self::MAX_EXECUTION_TIME);
|
set_time_limit(self::MAX_EXECUTION_TIME);
|
||||||
$this->processedCount = 0;
|
$this->processedCount = 0;
|
||||||
$this->errorCount = 0;
|
$this->errorCount = 0;
|
||||||
$this->transferBatch = [];
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private function processPeriod()
|
private function processPeriod()
|
||||||
@@ -114,23 +111,10 @@
|
|||||||
|
|
||||||
$headers = (new TempFundsTransfer())->getFillable();
|
$headers = (new TempFundsTransfer())->getFillable();
|
||||||
$rowCount = 0;
|
$rowCount = 0;
|
||||||
$chunkCount = 0;
|
|
||||||
|
|
||||||
while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) {
|
while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) {
|
||||||
$rowCount++;
|
$rowCount++;
|
||||||
$this->processRow($row, $headers, $rowCount, $filePath);
|
$this->processRow($row, $headers, $rowCount, $filePath);
|
||||||
|
|
||||||
// Process in chunks to avoid memory issues
|
|
||||||
if (count($this->transferBatch) >= self::CHUNK_SIZE) {
|
|
||||||
$this->saveBatch();
|
|
||||||
$chunkCount++;
|
|
||||||
Log::info("Processed chunk $chunkCount ({$this->processedCount} records so far)");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Process any remaining records
|
|
||||||
if (!empty($this->transferBatch)) {
|
|
||||||
$this->saveBatch();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fclose($handle);
|
fclose($handle);
|
||||||
@@ -152,24 +136,18 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
$data = array_combine($headers, $row);
|
$data = array_combine($headers, $row);
|
||||||
$this->addToBatch($data, $rowCount, $filePath);
|
$this->saveRecord($data, $rowCount, $filePath);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
private function saveRecord(array $data, int $rowCount, string $filePath)
|
||||||
* Add record to batch instead of saving immediately
|
|
||||||
*/
|
|
||||||
private function addToBatch(array $data, int $rowCount, string $filePath)
|
|
||||||
: void
|
: void
|
||||||
{
|
{
|
||||||
try {
|
try {
|
||||||
if (isset($data['_id']) && $data['_id'] !== '_id') {
|
if (isset($data['_id']) && $data['_id'] !== '_id') {
|
||||||
// Add timestamp fields
|
TempFundsTransfer::updateOrCreate(
|
||||||
$now = now();
|
['_id' => $data['_id']],
|
||||||
$data['created_at'] = $now;
|
$data
|
||||||
$data['updated_at'] = $now;
|
);
|
||||||
|
|
||||||
// Add to transfer batch
|
|
||||||
$this->transferBatch[] = $data;
|
|
||||||
$this->processedCount++;
|
$this->processedCount++;
|
||||||
}
|
}
|
||||||
} catch (Exception $e) {
|
} catch (Exception $e) {
|
||||||
@@ -178,37 +156,6 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Save batched records to the database
|
|
||||||
*/
|
|
||||||
private function saveBatch()
|
|
||||||
: void
|
|
||||||
{
|
|
||||||
try {
|
|
||||||
if (!empty($this->transferBatch)) {
|
|
||||||
// Process in smaller chunks for better memory management
|
|
||||||
foreach ($this->transferBatch as $entry) {
|
|
||||||
// Extract all stmt_entry_ids from the current chunk
|
|
||||||
$entryIds = array_column($entry, '_id');
|
|
||||||
|
|
||||||
// Delete existing records with these IDs to avoid conflicts
|
|
||||||
TempFundsTransfer::whereIn('_id', $entryIds)->delete();
|
|
||||||
|
|
||||||
// Insert all records in the chunk at once
|
|
||||||
TempFundsTransfer::insert($entry);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Reset entry batch after processing
|
|
||||||
$this->transferBatch = [];
|
|
||||||
}
|
|
||||||
} catch (Exception $e) {
|
|
||||||
Log::error("Error in saveBatch: " . $e->getMessage());
|
|
||||||
$this->errorCount += count($this->transferBatch);
|
|
||||||
// Reset batch even if there's an error to prevent reprocessing the same failed records
|
|
||||||
$this->transferBatch = [];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private function cleanup(string $tempFilePath)
|
private function cleanup(string $tempFilePath)
|
||||||
: void
|
: void
|
||||||
{
|
{
|
||||||
|
|||||||
Reference in New Issue
Block a user