From b98408a8d234672d6d0723a75e9106cdc1b85ce6 Mon Sep 17 00:00:00 2001 From: Daeng Deni Mardaeni Date: Thu, 22 May 2025 17:36:55 +0700 Subject: [PATCH] feat(webstatement): optimalkan proses ekspor statement dengan chunking dan tabel terproses - Menambahkan penggunaan chunking dalam pemrosesan data `StmtEntry` untuk mengurangi konsumsi memori - Menghapus data yang telah diproses pada tabel `processed_statements` sebelum memproses ulang data baru - Menambahkan metode `processStatementData` untuk memproses data transaksi dengan pengelolaan hemisan-memori - Mengganti mekanisme ekspor CSV agar sesuai dengan data yang sudah diproses; menggunakan chunk pada pembacaan dan penulisan data agar lebih efisien - Memperkenalkan tabel baru `processed_statements` untuk menyimpan hasil pemrosesan sebagai cache sementara - Menambahkan log untuk setiap tahap pemrosesan untuk mempermudah debugging dan pelacakan proses - Menambahkan file model `ProcessedStatement` untuk interaksi dengan tabel `processed_statements` - Menambahkan file migrasi untuk membuat tabel `processed_statements` di database Enhancement ini meningkatkan performa aplikasi, terutama saat menangani data besar, dengan mengurangi penggunaan memori dan meningkatkan efisiensi proses ekspor. Signed-off-by: Daeng Deni Mardaeni --- app/Jobs/ExportStatementJob.php | 152 ++++++++++++------ app/Models/ProcessedStatement.php | 21 +++ ...3235_create_processed_statements_table.php | 41 +++++ 3 files changed, 162 insertions(+), 52 deletions(-) create mode 100644 app/Models/ProcessedStatement.php create mode 100644 database/migrations/2025_05_22_103235_create_processed_statements_table.php diff --git a/app/Jobs/ExportStatementJob.php b/app/Jobs/ExportStatementJob.php index a18098b..d9e3a3f 100644 --- a/app/Jobs/ExportStatementJob.php +++ b/app/Jobs/ExportStatementJob.php @@ -9,8 +9,10 @@ use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; + use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Storage; + use Modules\Webstatement\Models\ProcessedStatement; use Modules\Webstatement\Models\StmtEntry; use Modules\Webstatement\Models\TempFundsTransfer; use Modules\Webstatement\Models\TempStmtNarrFormat; @@ -25,6 +27,7 @@ protected $saldo; protected $disk; protected $fileName; + protected $chunkSize = 500; // Proses data dalam chunk untuk mengurangi penggunaan memori /** * Create a new job instance. @@ -52,9 +55,18 @@ try { Log::info("Starting export statement job for account: {$this->account_number}, period: {$this->period}"); - $stmt = $this->getStatementData(); - $mappedData = $this->mapStatementData($stmt); - $this->exportToCsv($mappedData); + // Cek apakah data sudah diproses sebelumnya + $existingData = ProcessedStatement::where('account_number', $this->account_number) + ->where('period', $this->period) + ->exists(); + + if (!$existingData) { + // Jika belum ada data yang diproses, lakukan pemrosesan + $this->processStatementData(); + } + + // Export data yang sudah diproses ke CSV + $this->exportToCsv(); Log::info("Export statement job completed successfully for account: {$this->account_number}, period: {$this->period}"); } catch (Exception $e) { @@ -64,49 +76,72 @@ } /** - * Get statement data from database + * Process statement data and save to database */ - private function getStatementData() + private function processStatementData() + : void { - return StmtEntry::with(['ft', 'transaction']) - ->where('account_number', $this->account_number) - ->where('booking_date', $this->period) - ->orderBy('date_time', 'ASC') - ->orderBy('trans_reference', 'ASC') - ->get(); - } + // Hapus data yang mungkin sudah ada untuk kombinasi account dan period yang sama + ProcessedStatement::where('account_number', $this->account_number) + ->where('period', $this->period) + ->delete(); - /** - * Map statement data to the required format - */ - private function mapStatementData($stmt) - { $runningBalance = (float) $this->saldo; + $totalCount = StmtEntry::where('account_number', $this->account_number) + ->where('booking_date', $this->period) + ->count(); - // Map the data to transform or format specific fields - $mappedData = $stmt->sortBy(['ACTUAL.DATE', 'REFERENCE.NUMBER']) - ->map(function ($item, $index) use (&$runningBalance) { - $runningBalance += (float) $item->amount_lcy; - return [ - 'NO' => 0, // Will be updated later - 'TRANSACTION.DATE' => Carbon::createFromFormat('YmdHi', $item->booking_date . substr($item->ft?->date_time ?? '0000000000', 6, 4)) - ->format('d/m/Y H:i'), - 'REFERENCE.NUMBER' => $item->trans_reference, - 'TRANSACTION.AMOUNT' => $item->amount_lcy, - 'TRANSACTION.TYPE' => $item->amount_lcy < 0 ? 'D' : 'C', - 'DESCRIPTION' => $this->generateNarrative($item), - 'END.BALANCE' => $runningBalance, - 'ACTUAL.DATE' => Carbon::createFromFormat('ymdHi', $item->ft?->date_time ?? '2505120000') - ->format('d/m/Y H:i'), - ]; - }) - ->values(); + Log::info("Processing {$totalCount} statement entries for account: {$this->account_number}"); - // Apply sequential numbers - return $mappedData->map(function ($item, $index) { - $item['NO'] = $index + 1; - return $item; - }); + // Proses data dalam chunk untuk mengurangi penggunaan memori + StmtEntry::with(['ft', 'transaction']) + ->where('account_number', $this->account_number) + ->where('booking_date', $this->period) + ->orderBy('date_time', 'ASC') + ->orderBy('trans_reference', 'ASC') + ->chunk($this->chunkSize, function ($entries) use (&$runningBalance) { + $processedData = []; + + foreach ($entries as $index => $item) { + $runningBalance += (float) $item->amount_lcy; + + try { + $transactionDate = Carbon::createFromFormat('YmdHi', $item->booking_date . substr($item->ft?->date_time ?? '0000000000', 6, 4)) + ->format('d/m/Y H:i'); + } catch (Exception $e) { + $transactionDate = Carbon::now()->format('d/m/Y H:i'); + Log::warning("Error formatting transaction date: " . $e->getMessage()); + } + + try { + $actualDate = Carbon::createFromFormat('ymdHi', $item->ft?->date_time ?? '2505120000') + ->format('d/m/Y H:i'); + } catch (Exception $e) { + $actualDate = Carbon::now()->format('d/m/Y H:i'); + Log::warning("Error formatting actual date: " . $e->getMessage()); + } + + $processedData[] = [ + 'account_number' => $this->account_number, + 'period' => $this->period, + 'sequence_no' => $index + 1, + 'transaction_date' => $transactionDate, + 'reference_number' => $item->trans_reference, + 'transaction_amount' => $item->amount_lcy, + 'transaction_type' => $item->amount_lcy < 0 ? 'D' : 'C', + 'description' => $this->generateNarrative($item), + 'end_balance' => $runningBalance, + 'actual_date' => $actualDate, + 'created_at' => now(), + 'updated_at' => now(), + ]; + } + + // Simpan data dalam batch untuk kinerja yang lebih baik + if (!empty($processedData)) { + DB::table('processed_statements')->insert($processedData); + } + }); } /** @@ -221,22 +256,35 @@ } /** - * Export data to CSV file + * Export processed data to CSV file */ - private function exportToCsv($mappedData) + private function exportToCsv() + : void { - $csvContent = ''; + $csvContent = "NO|TRANSACTION.DATE|REFERENCE.NUMBER|TRANSACTION.AMOUNT|TRANSACTION.TYPE|DESCRIPTION|END.BALANCE|ACTUAL.DATE\n"; - // Add headers - $csvContent .= implode('|', array_keys($mappedData[0])) . "\n"; + // Ambil data yang sudah diproses dalam chunk untuk mengurangi penggunaan memori + ProcessedStatement::where('account_number', $this->account_number) + ->where('period', $this->period) + ->orderBy('sequence_no') + ->chunk($this->chunkSize, function ($statements) use (&$csvContent) { + foreach ($statements as $statement) { + $csvContent .= implode('|', [ + $statement->sequence_no, + $statement->transaction_date, + $statement->reference_number, + $statement->transaction_amount, + $statement->transaction_type, + $statement->description, + $statement->end_balance, + $statement->actual_date + ]) . "\n"; + } - // Add data rows - foreach ($mappedData as $row) { - $csvContent .= implode('|', $row) . "\n"; - } - - // Save to storage - Storage::disk($this->disk)->put("statements/{$this->fileName}", $csvContent); + // Tulis ke file secara bertahap untuk mengurangi penggunaan memori + Storage::disk($this->disk)->append("statements/{$this->fileName}", $csvContent); + $csvContent = ''; // Reset content setelah ditulis + }); Log::info("Statement exported to {$this->disk} disk: statements/{$this->fileName}"); } diff --git a/app/Models/ProcessedStatement.php b/app/Models/ProcessedStatement.php new file mode 100644 index 0000000..b199add --- /dev/null +++ b/app/Models/ProcessedStatement.php @@ -0,0 +1,21 @@ +id(); + $table->string('account_number'); + $table->string('period'); + $table->integer('sequence_no'); + $table->string('transaction_date'); + $table->string('reference_number'); + $table->decimal('transaction_amount', 20, 2); + $table->char('transaction_type', 1); + $table->text('description')->nullable(); + $table->decimal('end_balance', 20, 2); + $table->string('actual_date'); + $table->timestamps(); + + // Indeks untuk pencarian cepat + $table->index(['account_number', 'period']); + $table->index('sequence_no'); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::dropIfExists('processed_statements'); + } + };