refactor(webstatement): optimize ProcessAccountDataJob logic with batch processing

- Menambahkan properti baru `CHUNK_SIZE` untuk mengelola proses data dalam batch sehingga mengurangi penggunaan memori.
- Memperkenalkan batching untuk penyimpanan akun (`accountBatch`) dan balance (`balanceBatch`) untuk mengurangi beban query database.
- Mengganti penyimpanan langsung di database dengan metode bulk insert menggunakan `upsert` pada tabel Account dan AccountBalance.
- Menambahkan logging untuk mencatat setiap pemrosesan batch dan memberikan deskripsi jumlah chunk yang telah diproses.
- Memindahkan proses normalization dan balance data ke sistem batching alih-alih penyimpanan langsung.
- Menambahkan validasi tambahan untuk pengolahan data row dan mencegah error data yang memiliki kolom tidak sesuai.
- Memperbarui log pesan untuk menampilkan jumlah record yang diproses dan error secara lebih rinci.
- Memodularisasi fungsi agar lebih readable dan maintainable dengan beberapa fungsi baru seperti `addToBatch` dan `saveBatch`.

app/Jobs/ProcessAccountDataJob.php

@@ -1,31 +1,35 @@
 <?php
 
-    namespace Modules\Webstatement\Jobs;
+namespace Modules\Webstatement\Jobs;
 
-    use Exception;
+use Exception;
-    use Illuminate\Bus\Queueable;
+use Illuminate\Bus\Queueable;
-    use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Contracts\Queue\ShouldQueue;
-    use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Foundation\Bus\Dispatchable;
-    use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\InteractsWithQueue;
-    use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\SerializesModels;
-    use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\Log;
-    use Illuminate\Support\Facades\Storage;
+use Illuminate\Support\Facades\Storage;
-    use Modules\Webstatement\Models\Account;
+use Modules\Webstatement\Models\Account;
-    use Modules\Webstatement\Models\AccountBalance;
+use Modules\Webstatement\Models\AccountBalance;
 
-    class ProcessAccountDataJob implements ShouldQueue
+class ProcessAccountDataJob implements ShouldQueue
-    {
+{
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
     private const CSV_DELIMITER      = '~';
     private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds
     private const FILENAME           = 'ST.ACCOUNT.csv';
     private const DISK_NAME          = 'sftpStatement';
+    private const CHUNK_SIZE         = 1000; // Process data in chunks to reduce memory usage
+
     private string $period = '';
     private int    $processedCount = 0;
     private int    $errorCount     = 0;
 
     private $balanceData = [];
+    private $accountBatch = [];
+    private $balanceBatch = [];
 
     /**
      * Create a new job instance.
@@ -63,6 +67,8 @@
         set_time_limit(self::MAX_EXECUTION_TIME);
         $this->processedCount = 0;
         $this->errorCount     = 0;
+        $this->accountBatch   = [];
+        $this->balanceBatch   = [];
     }
 
     private function processPeriod()
@@ -114,10 +120,23 @@
         $headers = (new Account())->getFillable();
         Log::info('Headers: ' . implode(", ", $headers));
         $rowCount = 0;
+        $chunkCount = 0;
 
         while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) {
             $rowCount++;
             $this->processRow($row, $headers, $rowCount, $filePath);
+
+            // Process in chunks to avoid memory issues
+            if (count($this->accountBatch) >= self::CHUNK_SIZE) {
+                $this->saveBatch();
+                $chunkCount++;
+                Log::info("Processed chunk $chunkCount ({$this->processedCount} records so far)");
+            }
+        }
+
+        // Process any remaining records
+        if (!empty($this->accountBatch) || !empty($this->balanceBatch)) {
+            $this->saveBatch();
         }
 
         fclose($handle);
@@ -135,7 +154,7 @@
 
         $data = array_combine($headers, $row);
         $this->normalizeData($data);
-            $this->saveRecord($data, $rowCount, $filePath);
+        $this->addToBatch($data, $rowCount, $filePath);
     }
 
     private function normalizeData(array &$data)
@@ -161,17 +180,34 @@
         unset($data['open_cleared_bal']);
     }
 
-        private function saveRecord(array $data, int $rowCount, string $filePath)
+    /**
+     * Add record to batch instead of saving immediately
+     */
+    private function addToBatch(array $data, int $rowCount, string $filePath)
     : void
     {
         try {
             if ($data['account_number'] !== 'account_number') {
-                    // Use firstOrNew instead of updateOrCreate
+                // Add timestamp fields
-                    $account = Account::firstOrNew(['account_number' => $data['account_number']]);
+                $now = now();
-                    $account->fill($data);
+                $data['created_at'] = $now;
-                    $account->save();
+                $data['updated_at'] = $now;
+
+                // Add to account batch
+                $this->accountBatch[] = $data;
+
+                // Add to balance batch
+                if (isset($this->balanceData['open_actual_bal']) || isset($this->balanceData['open_cleared_bal'])) {
+                    $this->balanceBatch[] = [
+                        'account_number'  => $data['account_number'],
+                        'period'          => $this->period,
+                        'actual_balance'  => $this->balanceData['open_actual_bal'],
+                        'cleared_balance' => $this->balanceData['open_cleared_bal'],
+                        'created_at'      => $now,
+                        'updated_at'      => $now
+                    ];
+                }
 
-                    $this->saveAccountBalance($data['account_number']);
                 $this->processedCount++;
             }
         } catch (Exception $e) {
@@ -180,29 +216,39 @@
         }
     }
 
-        private function saveAccountBalance(string $accountNumber)
+    /**
+     * Save batched records to the database
+     */
+    private function saveBatch()
     : void
     {
-            // Store the opening balances in the AccountBalance model for this period
+        try {
-            if (isset($this->balanceData['open_actual_bal']) || isset($this->balanceData['open_cleared_bal'])) {
+            if (!empty($this->accountBatch)) {
-                // Prepare balance data for bulk insert/update
+                // Bulk insert/update accounts
-                $balanceData = [
+                Account::upsert(
-                    'account_number'  => $accountNumber,
+                    $this->accountBatch,
-                    'period'          => $this->period,
+                    ['account_number'], // Unique key
-                    'actual_balance'  => $this->balanceData['open_actual_bal'],
+                    array_diff((new Account())->getFillable(), ['account_number']) // Update columns
-                    'cleared_balance' => $this->balanceData['open_cleared_bal'],
-                    'created_at'      => now(),
-                    'updated_at'      => now()
-                ];
-
-                // Use updateOrInsert to reduce queries
-                AccountBalance::updateOrInsert(
-                    [
-                        'account_number' => $accountNumber,
-                        'period'         => $this->period
-                    ],
-                    $balanceData
                 );
+
+                // Reset account batch after processing
+                $this->accountBatch = [];
+            }
+
+            if (!empty($this->balanceBatch)) {
+                // Bulk insert/update account balances
+                AccountBalance::upsert(
+                    $this->balanceBatch,
+                    ['account_number', 'period'], // Composite unique key
+                    ['actual_balance', 'cleared_balance', 'updated_at'] // Update columns
+                );
+
+                // Reset balance batch after processing
+                $this->balanceBatch = [];
+            }
+        } catch (Exception $e) {
+            Log::error("Error in saveBatch: " . $e->getMessage());
+            $this->errorCount += count($this->accountBatch);
         }
     }
 
@@ -220,4 +266,4 @@
         Log::info("Account data processing completed. " .
             "Total processed: {$this->processedCount}, Total errors: {$this->errorCount}");
     }
-    }
+}