daengdeni/webstatement
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(webstatement): optimalkan proses ekspor statement dengan chunking dan tabel terproses

Browse Source 
- Menambahkan penggunaan chunking dalam pemrosesan data `StmtEntry` untuk mengurangi konsumsi memori
- Menghapus data yang telah diproses pada tabel `processed_statements` sebelum memproses ulang data baru
- Menambahkan metode `processStatementData` untuk memproses data transaksi dengan pengelolaan hemisan-memori
- Mengganti mekanisme ekspor CSV agar sesuai dengan data yang sudah diproses; menggunakan chunk pada pembacaan dan penulisan data agar lebih efisien
- Memperkenalkan tabel baru `processed_statements` untuk menyimpan hasil pemrosesan sebagai cache sementara
- Menambahkan log untuk setiap tahap pemrosesan untuk mempermudah debugging dan pelacakan proses
- Menambahkan file model `ProcessedStatement` untuk interaksi dengan tabel `processed_statements`
- Menambahkan file migrasi untuk membuat tabel `processed_statements` di database

Enhancement ini meningkatkan performa aplikasi, terutama saat menangani data besar, dengan mengurangi penggunaan memori dan meningkatkan efisiensi proses ekspor.

Signed-off-by: Daeng Deni Mardaeni <ddeni05@gmail.com>
This commit is contained in:
Daeng Deni Mardaeni
2025-05-22 17:36:55 +07:00
parent 04f6f02702
commit b98408a8d2
3 changed files with 162 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

138
app/Jobs/ExportStatementJob.php
View File

@@ -9,8 +9,10 @@
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
use Modules\Webstatement\Models\ProcessedStatement;
use Modules\Webstatement\Models\StmtEntry;
use Modules\Webstatement\Models\TempFundsTransfer;
use Modules\Webstatement\Models\TempStmtNarrFormat;
@@ -25,6 +27,7 @@
protected $saldo;
protected $disk;
protected $fileName;
protected $chunkSize = 500; // Proses data dalam chunk untuk mengurangi penggunaan memori
/**
* Create a new job instance.
@@ -52,9 +55,18 @@
try {
Log::info("Starting export statement job for account: {$this->account_number}, period: {$this->period}");
$stmt = $this->getStatementData();
$mappedData = $this->mapStatementData($stmt);
$this->exportToCsv($mappedData);
// Cek apakah data sudah diproses sebelumnya
$existingData = ProcessedStatement::where('account_number', $this->account_number)
->where('period', $this->period)
->exists();
if (!$existingData) {
// Jika belum ada data yang diproses, lakukan pemrosesan
$this->processStatementData();
}
// Export data yang sudah diproses ke CSV
$this->exportToCsv();
Log::info("Export statement job completed successfully for account: {$this->account_number}, period: {$this->period}");
} catch (Exception $e) {
@@ -64,48 +76,71 @@
}
/**
* Get statement data from database
* Process statement data and save to database
*/
private function getStatementData()
private function processStatementData()
: void
{
return StmtEntry::with(['ft', 'transaction'])
// Hapus data yang mungkin sudah ada untuk kombinasi account dan period yang sama
ProcessedStatement::where('account_number', $this->account_number)
->where('period', $this->period)
->delete();
$runningBalance = (float) $this->saldo;
$totalCount = StmtEntry::where('account_number', $this->account_number)
->where('booking_date', $this->period)
->count();
Log::info("Processing {$totalCount} statement entries for account: {$this->account_number}");
// Proses data dalam chunk untuk mengurangi penggunaan memori
StmtEntry::with(['ft', 'transaction'])
->where('account_number', $this->account_number)
->where('booking_date', $this->period)
->orderBy('date_time', 'ASC')
->orderBy('trans_reference', 'ASC')
->get();
->chunk($this->chunkSize, function ($entries) use (&$runningBalance) {
$processedData = [];
foreach ($entries as $index => $item) {
$runningBalance += (float) $item->amount_lcy;
try {
$transactionDate = Carbon::createFromFormat('YmdHi', $item->booking_date . substr($item->ft?->date_time ?? '0000000000', 6, 4))
->format('d/m/Y H:i');
} catch (Exception $e) {
$transactionDate = Carbon::now()->format('d/m/Y H:i');
Log::warning("Error formatting transaction date: " . $e->getMessage());
}
/**
* Map statement data to the required format
*/
private function mapStatementData($stmt)
{
$runningBalance = (float) $this->saldo;
try {
$actualDate = Carbon::createFromFormat('ymdHi', $item->ft?->date_time ?? '2505120000')
->format('d/m/Y H:i');
} catch (Exception $e) {
$actualDate = Carbon::now()->format('d/m/Y H:i');
Log::warning("Error formatting actual date: " . $e->getMessage());
}
// Map the data to transform or format specific fields
$mappedData = $stmt->sortBy(['ACTUAL.DATE', 'REFERENCE.NUMBER'])
->map(function ($item, $index) use (&$runningBalance) {
$runningBalance += (float) $item->amount_lcy;
return [
'NO' => 0, // Will be updated later
'TRANSACTION.DATE' => Carbon::createFromFormat('YmdHi', $item->booking_date . substr($item->ft?->date_time ?? '0000000000', 6, 4))
->format('d/m/Y H:i'),
'REFERENCE.NUMBER' => $item->trans_reference,
'TRANSACTION.AMOUNT' => $item->amount_lcy,
'TRANSACTION.TYPE' => $item->amount_lcy < 0 ? 'D' : 'C',
'DESCRIPTION' => $this->generateNarrative($item),
'END.BALANCE' => $runningBalance,
'ACTUAL.DATE' => Carbon::createFromFormat('ymdHi', $item->ft?->date_time ?? '2505120000')
->format('d/m/Y H:i'),
$processedData[] = [
'account_number' => $this->account_number,
'period' => $this->period,
'sequence_no' => $index + 1,
'transaction_date' => $transactionDate,
'reference_number' => $item->trans_reference,
'transaction_amount' => $item->amount_lcy,
'transaction_type' => $item->amount_lcy < 0 ? 'D' : 'C',
'description' => $this->generateNarrative($item),
'end_balance' => $runningBalance,
'actual_date' => $actualDate,
'created_at' => now(),
'updated_at' => now(),
];
})
->values();
}
// Apply sequential numbers
return $mappedData->map(function ($item, $index) {
$item['NO'] = $index + 1;
return $item;
// Simpan data dalam batch untuk kinerja yang lebih baik
if (!empty($processedData)) {
DB::table('processed_statements')->insert($processedData);
}
});
}
@@ -221,22 +256,35 @@
}
/**
* Export data to CSV file
* Export processed data to CSV file
*/
private function exportToCsv($mappedData)
private function exportToCsv()
: void
{
$csvContent = '';
$csvContent = "NO|TRANSACTION.DATE|REFERENCE.NUMBER|TRANSACTION.AMOUNT|TRANSACTION.TYPE|DESCRIPTION|END.BALANCE|ACTUAL.DATE\n";
// Add headers
$csvContent .= implode('|', array_keys($mappedData[0])) . "\n";
// Add data rows
foreach ($mappedData as $row) {
$csvContent .= implode('|', $row) . "\n";
// Ambil data yang sudah diproses dalam chunk untuk mengurangi penggunaan memori
ProcessedStatement::where('account_number', $this->account_number)
->where('period', $this->period)
->orderBy('sequence_no')
->chunk($this->chunkSize, function ($statements) use (&$csvContent) {
foreach ($statements as $statement) {
$csvContent .= implode('|', [
$statement->sequence_no,
$statement->transaction_date,
$statement->reference_number,
$statement->transaction_amount,
$statement->transaction_type,
$statement->description,
$statement->end_balance,
$statement->actual_date
]) . "\n";
}
// Save to storage
Storage::disk($this->disk)->put("statements/{$this->fileName}", $csvContent);
// Tulis ke file secara bertahap untuk mengurangi penggunaan memori
Storage::disk($this->disk)->append("statements/{$this->fileName}", $csvContent);
$csvContent = ''; // Reset content setelah ditulis
});
Log::info("Statement exported to {$this->disk} disk: statements/{$this->fileName}");
}

21
app/Models/ProcessedStatement.php Normal file
View File

@@ -0,0 +1,21 @@
<?php
namespace Modules\Webstatement\Models;
use Illuminate\Database\Eloquent\Model;
class ProcessedStatement extends Model
{
protected $fillable = [
'account_number',
'period',
'sequence_no',
'transaction_date',
'reference_number',
'transaction_amount',
'transaction_type',
'description',
'end_balance',
'actual_date'
];
}

41
database/migrations/2025_05_22_103235_create_processed_statements_table.php Normal file
View File

@@ -0,0 +1,41 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::create('processed_statements', function (Blueprint $table) {
$table->id();
$table->string('account_number');
$table->string('period');
$table->integer('sequence_no');
$table->string('transaction_date');
$table->string('reference_number');
$table->decimal('transaction_amount', 20, 2);
$table->char('transaction_type', 1);
$table->text('description')->nullable();
$table->decimal('end_balance', 20, 2);
$table->string('actual_date');
$table->timestamps();
// Indeks untuk pencarian cepat
$table->index(['account_number', 'period']);
$table->index('sequence_no');
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::dropIfExists('processed_statements');
}
};