feat(webstatement): optimalkan proses ekspor statement dengan chunking dan tabel terproses
- Menambahkan penggunaan chunking dalam pemrosesan data `StmtEntry` untuk mengurangi konsumsi memori - Menghapus data yang telah diproses pada tabel `processed_statements` sebelum memproses ulang data baru - Menambahkan metode `processStatementData` untuk memproses data transaksi dengan pengelolaan hemisan-memori - Mengganti mekanisme ekspor CSV agar sesuai dengan data yang sudah diproses; menggunakan chunk pada pembacaan dan penulisan data agar lebih efisien - Memperkenalkan tabel baru `processed_statements` untuk menyimpan hasil pemrosesan sebagai cache sementara - Menambahkan log untuk setiap tahap pemrosesan untuk mempermudah debugging dan pelacakan proses - Menambahkan file model `ProcessedStatement` untuk interaksi dengan tabel `processed_statements` - Menambahkan file migrasi untuk membuat tabel `processed_statements` di database Enhancement ini meningkatkan performa aplikasi, terutama saat menangani data besar, dengan mengurangi penggunaan memori dan meningkatkan efisiensi proses ekspor. Signed-off-by: Daeng Deni Mardaeni <ddeni05@gmail.com>
This commit is contained in:
Daeng Deni Mardaeni
@@ -9,8 +9,10 @@
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use Modules\Webstatement\Models\ProcessedStatement;
|
||||
use Modules\Webstatement\Models\StmtEntry;
|
||||
use Modules\Webstatement\Models\TempFundsTransfer;
|
||||
use Modules\Webstatement\Models\TempStmtNarrFormat;
|
||||
@@ -25,6 +27,7 @@
|
||||
protected $saldo;
|
||||
protected $disk;
|
||||
protected $fileName;
|
||||
protected $chunkSize = 500; // Proses data dalam chunk untuk mengurangi penggunaan memori
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
@@ -52,9 +55,18 @@
|
||||
try {
|
||||
Log::info("Starting export statement job for account: {$this->account_number}, period: {$this->period}");
|
||||
|
||||
$stmt = $this->getStatementData();
|
||||
$mappedData = $this->mapStatementData($stmt);
|
||||
$this->exportToCsv($mappedData);
|
||||
// Cek apakah data sudah diproses sebelumnya
|
||||
$existingData = ProcessedStatement::where('account_number', $this->account_number)
|
||||
->where('period', $this->period)
|
||||
->exists();
|
||||
|
||||
if (!$existingData) {
|
||||
// Jika belum ada data yang diproses, lakukan pemrosesan
|
||||
$this->processStatementData();
|
||||
}
|
||||
|
||||
// Export data yang sudah diproses ke CSV
|
||||
$this->exportToCsv();
|
||||
|
||||
Log::info("Export statement job completed successfully for account: {$this->account_number}, period: {$this->period}");
|
||||
} catch (Exception $e) {
|
||||
@@ -64,49 +76,72 @@
|
||||
}
|
||||
|
||||
/**
|
||||
* Get statement data from database
|
||||
* Process statement data and save to database
|
||||
*/
|
||||
private function getStatementData()
|
||||
private function processStatementData()
|
||||
: void
|
||||
{
|
||||
return StmtEntry::with(['ft', 'transaction'])
|
||||
->where('account_number', $this->account_number)
|
||||
->where('booking_date', $this->period)
|
||||
->orderBy('date_time', 'ASC')
|
||||
->orderBy('trans_reference', 'ASC')
|
||||
->get();
|
||||
}
|
||||
// Hapus data yang mungkin sudah ada untuk kombinasi account dan period yang sama
|
||||
ProcessedStatement::where('account_number', $this->account_number)
|
||||
->where('period', $this->period)
|
||||
->delete();
|
||||
|
||||
/**
|
||||
* Map statement data to the required format
|
||||
*/
|
||||
private function mapStatementData($stmt)
|
||||
{
|
||||
$runningBalance = (float) $this->saldo;
|
||||
$totalCount = StmtEntry::where('account_number', $this->account_number)
|
||||
->where('booking_date', $this->period)
|
||||
->count();
|
||||
|
||||
// Map the data to transform or format specific fields
|
||||
$mappedData = $stmt->sortBy(['ACTUAL.DATE', 'REFERENCE.NUMBER'])
|
||||
->map(function ($item, $index) use (&$runningBalance) {
|
||||
$runningBalance += (float) $item->amount_lcy;
|
||||
return [
|
||||
'NO' => 0, // Will be updated later
|
||||
'TRANSACTION.DATE' => Carbon::createFromFormat('YmdHi', $item->booking_date . substr($item->ft?->date_time ?? '0000000000', 6, 4))
|
||||
->format('d/m/Y H:i'),
|
||||
'REFERENCE.NUMBER' => $item->trans_reference,
|
||||
'TRANSACTION.AMOUNT' => $item->amount_lcy,
|
||||
'TRANSACTION.TYPE' => $item->amount_lcy < 0 ? 'D' : 'C',
|
||||
'DESCRIPTION' => $this->generateNarrative($item),
|
||||
'END.BALANCE' => $runningBalance,
|
||||
'ACTUAL.DATE' => Carbon::createFromFormat('ymdHi', $item->ft?->date_time ?? '2505120000')
|
||||
->format('d/m/Y H:i'),
|
||||
];
|
||||
})
|
||||
->values();
|
||||
Log::info("Processing {$totalCount} statement entries for account: {$this->account_number}");
|
||||
|
||||
// Apply sequential numbers
|
||||
return $mappedData->map(function ($item, $index) {
|
||||
$item['NO'] = $index + 1;
|
||||
return $item;
|
||||
});
|
||||
// Proses data dalam chunk untuk mengurangi penggunaan memori
|
||||
StmtEntry::with(['ft', 'transaction'])
|
||||
->where('account_number', $this->account_number)
|
||||
->where('booking_date', $this->period)
|
||||
->orderBy('date_time', 'ASC')
|
||||
->orderBy('trans_reference', 'ASC')
|
||||
->chunk($this->chunkSize, function ($entries) use (&$runningBalance) {
|
||||
$processedData = [];
|
||||
|
||||
foreach ($entries as $index => $item) {
|
||||
$runningBalance += (float) $item->amount_lcy;
|
||||
|
||||
try {
|
||||
$transactionDate = Carbon::createFromFormat('YmdHi', $item->booking_date . substr($item->ft?->date_time ?? '0000000000', 6, 4))
|
||||
->format('d/m/Y H:i');
|
||||
} catch (Exception $e) {
|
||||
$transactionDate = Carbon::now()->format('d/m/Y H:i');
|
||||
Log::warning("Error formatting transaction date: " . $e->getMessage());
|
||||
}
|
||||
|
||||
try {
|
||||
$actualDate = Carbon::createFromFormat('ymdHi', $item->ft?->date_time ?? '2505120000')
|
||||
->format('d/m/Y H:i');
|
||||
} catch (Exception $e) {
|
||||
$actualDate = Carbon::now()->format('d/m/Y H:i');
|
||||
Log::warning("Error formatting actual date: " . $e->getMessage());
|
||||
}
|
||||
|
||||
$processedData[] = [
|
||||
'account_number' => $this->account_number,
|
||||
'period' => $this->period,
|
||||
'sequence_no' => $index + 1,
|
||||
'transaction_date' => $transactionDate,
|
||||
'reference_number' => $item->trans_reference,
|
||||
'transaction_amount' => $item->amount_lcy,
|
||||
'transaction_type' => $item->amount_lcy < 0 ? 'D' : 'C',
|
||||
'description' => $this->generateNarrative($item),
|
||||
'end_balance' => $runningBalance,
|
||||
'actual_date' => $actualDate,
|
||||
'created_at' => now(),
|
||||
'updated_at' => now(),
|
||||
];
|
||||
}
|
||||
|
||||
// Simpan data dalam batch untuk kinerja yang lebih baik
|
||||
if (!empty($processedData)) {
|
||||
DB::table('processed_statements')->insert($processedData);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -221,22 +256,35 @@
|
||||
}
|
||||
|
||||
/**
|
||||
* Export data to CSV file
|
||||
* Export processed data to CSV file
|
||||
*/
|
||||
private function exportToCsv($mappedData)
|
||||
private function exportToCsv()
|
||||
: void
|
||||
{
|
||||
$csvContent = '';
|
||||
$csvContent = "NO|TRANSACTION.DATE|REFERENCE.NUMBER|TRANSACTION.AMOUNT|TRANSACTION.TYPE|DESCRIPTION|END.BALANCE|ACTUAL.DATE\n";
|
||||
|
||||
// Add headers
|
||||
$csvContent .= implode('|', array_keys($mappedData[0])) . "\n";
|
||||
// Ambil data yang sudah diproses dalam chunk untuk mengurangi penggunaan memori
|
||||
ProcessedStatement::where('account_number', $this->account_number)
|
||||
->where('period', $this->period)
|
||||
->orderBy('sequence_no')
|
||||
->chunk($this->chunkSize, function ($statements) use (&$csvContent) {
|
||||
foreach ($statements as $statement) {
|
||||
$csvContent .= implode('|', [
|
||||
$statement->sequence_no,
|
||||
$statement->transaction_date,
|
||||
$statement->reference_number,
|
||||
$statement->transaction_amount,
|
||||
$statement->transaction_type,
|
||||
$statement->description,
|
||||
$statement->end_balance,
|
||||
$statement->actual_date
|
||||
]) . "\n";
|
||||
}
|
||||
|
||||
// Add data rows
|
||||
foreach ($mappedData as $row) {
|
||||
$csvContent .= implode('|', $row) . "\n";
|
||||
}
|
||||
|
||||
// Save to storage
|
||||
Storage::disk($this->disk)->put("statements/{$this->fileName}", $csvContent);
|
||||
// Tulis ke file secara bertahap untuk mengurangi penggunaan memori
|
||||
Storage::disk($this->disk)->append("statements/{$this->fileName}", $csvContent);
|
||||
$csvContent = ''; // Reset content setelah ditulis
|
||||
});
|
||||
|
||||
Log::info("Statement exported to {$this->disk} disk: statements/{$this->fileName}");
|
||||
}
|
||||
|
||||
21
app/Models/ProcessedStatement.php
Normal file
21
app/Models/ProcessedStatement.php
Normal file
@@ -0,0 +1,21 @@
|
||||
<?php
|
||||
|
||||
namespace Modules\Webstatement\Models;
|
||||
|
||||
use Illuminate\Database\Eloquent\Model;
|
||||
|
||||
class ProcessedStatement extends Model
|
||||
{
|
||||
protected $fillable = [
|
||||
'account_number',
|
||||
'period',
|
||||
'sequence_no',
|
||||
'transaction_date',
|
||||
'reference_number',
|
||||
'transaction_amount',
|
||||
'transaction_type',
|
||||
'description',
|
||||
'end_balance',
|
||||
'actual_date'
|
||||
];
|
||||
}
|
||||
Reference in New Issue
Block a user