feat(webstatement): tambah stmt_entry_detail migrasi, model, dan job processing
Menambahkan fitur pengelolaan data stmt_entry_detail untuk integrasi transaksi dengan detail yang lebih lengkap. Perubahan yang dilakukan: - Membuat migrasi create_stmt_entry_detail_table dengan struktur field sesuai kebutuhan bisnis - Menambahkan index pada kolom penting untuk meningkatkan performa query - Membuat model StmtEntryDetail dengan relasi ke: - Account - TempFundsTransfer - TempTransaction - Teller - DataCapture - TempArrangement - Mengimplementasikan $fillable dan $casts sesuai struktur tabel - Menambahkan relasi untuk memudahkan integrasi antar modul - Membuat job ProcessStmtEntryDetailDataJob untuk memproses file CSV dengan batch processing - Mengimplementasikan chunking untuk menangani file besar secara efisien - Membersihkan trans_reference dari karakter tidak valid sebelum penyimpanan - Menggunakan updateOrCreate untuk mencegah duplikasi primary key - Menggunakan database transaction untuk menjaga konsistensi data - Menambahkan logging komprehensif untuk monitoring dan debugging - Mengimplementasikan error handling yang robust untuk menghindari job failure tanpa informasi - Memastikan penggunaan resource memory tetap optimal saat memproses data besar - Menambahkan case baru di MigrasiController untuk memproses stmt_entry_detail - Konsisten dengan pattern migrasi data yang sudah ada di sistem Tujuan perubahan: - Menyediakan sistem import dan pengolahan data stmt_entry_detail dengan proses yang aman dan efisien - Memudahkan integrasi transaksi dengan detail tambahan di modul Webstatement - Menjamin integritas data dengan penggunaan transaction, logging, dan error handling yang komprehensif
This commit is contained in:
Daeng Deni Mardaeni
@@ -191,11 +191,11 @@
|
||||
$subQuery->where('product_code', '!=', '6021')
|
||||
->orWhere(function($nestedQuery) {
|
||||
$nestedQuery->where('product_code', '6021')
|
||||
->where('ctdesc', '!=', 'gold');
|
||||
->where('ctdesc', '!=', 'GOLD');
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
|
||||
|
||||
$cards = $query->get();
|
||||
|
||||
|
||||
340
app/Jobs/ProcessStmtEntryDetailDataJob.php
Normal file
340
app/Jobs/ProcessStmtEntryDetailDataJob.php
Normal file
@@ -0,0 +1,340 @@
|
||||
<?php
|
||||
|
||||
namespace Modules\Webstatement\Jobs;
|
||||
|
||||
use Exception;
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use Modules\Webstatement\Models\StmtEntryDetail;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
|
||||
class ProcessStmtEntryDetailDataJob implements ShouldQueue
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
private const CSV_DELIMITER = '~';
|
||||
private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds
|
||||
private const FILENAME = 'ST.STMT.ENTRY.DETAIL.csv';
|
||||
private const DISK_NAME = 'sftpStatement';
|
||||
private const CHUNK_SIZE = 1000; // Process data in chunks to reduce memory usage
|
||||
|
||||
private string $period = '';
|
||||
private int $processedCount = 0;
|
||||
private int $errorCount = 0;
|
||||
private array $entryBatch = [];
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
*
|
||||
* @param string $period Periode data yang akan diproses
|
||||
*/
|
||||
public function __construct(string $period = '')
|
||||
{
|
||||
$this->period = $period;
|
||||
Log::info('ProcessStmtEntryDetailDataJob initialized', ['period' => $period]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the job.
|
||||
*
|
||||
* @return void
|
||||
* @throws Exception
|
||||
*/
|
||||
public function handle(): void
|
||||
{
|
||||
try {
|
||||
Log::info('Memulai ProcessStmtEntryDetailDataJob', ['period' => $this->period]);
|
||||
|
||||
$this->initializeJob();
|
||||
|
||||
if ($this->period === '') {
|
||||
Log::warning('No period provided for statement entry detail data processing');
|
||||
return;
|
||||
}
|
||||
|
||||
$this->processPeriod();
|
||||
$this->logJobCompletion();
|
||||
|
||||
Log::info('ProcessStmtEntryDetailDataJob selesai berhasil');
|
||||
} catch (Exception $e) {
|
||||
Log::error('Error in ProcessStmtEntryDetailDataJob: ' . $e->getMessage());
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Inisialisasi job dengan pengaturan awal
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
private function initializeJob(): void
|
||||
{
|
||||
set_time_limit(self::MAX_EXECUTION_TIME);
|
||||
$this->processedCount = 0;
|
||||
$this->errorCount = 0;
|
||||
$this->entryBatch = [];
|
||||
|
||||
Log::info('Job initialized', [
|
||||
'max_execution_time' => self::MAX_EXECUTION_TIME,
|
||||
'chunk_size' => self::CHUNK_SIZE
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Proses data untuk periode tertentu
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
private function processPeriod(): void
|
||||
{
|
||||
$disk = Storage::disk(self::DISK_NAME);
|
||||
$filename = "{$this->period}." . self::FILENAME;
|
||||
$filePath = "{$this->period}/$filename";
|
||||
|
||||
Log::info('Memulai proses periode', ['file_path' => $filePath]);
|
||||
|
||||
if (!$this->validateFile($disk, $filePath)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$tempFilePath = $this->createTemporaryFile($disk, $filePath, $filename);
|
||||
$this->processFile($tempFilePath, $filePath);
|
||||
$this->cleanup($tempFilePath);
|
||||
|
||||
Log::info('Proses periode selesai', ['file_path' => $filePath]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validasi keberadaan file
|
||||
*
|
||||
* @param mixed $disk Storage disk instance
|
||||
* @param string $filePath Path file yang akan divalidasi
|
||||
* @return bool
|
||||
*/
|
||||
private function validateFile($disk, string $filePath): bool
|
||||
{
|
||||
Log::info("Processing statement entry detail file: $filePath");
|
||||
|
||||
if (!$disk->exists($filePath)) {
|
||||
Log::warning("File not found: $filePath");
|
||||
return false;
|
||||
}
|
||||
|
||||
Log::info("File validated successfully: $filePath");
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Buat file temporary untuk proses
|
||||
*
|
||||
* @param mixed $disk Storage disk instance
|
||||
* @param string $filePath Path file sumber
|
||||
* @param string $filename Nama file
|
||||
* @return string Path file temporary
|
||||
*/
|
||||
private function createTemporaryFile($disk, string $filePath, string $filename): string
|
||||
{
|
||||
$tempFilePath = storage_path("app/temp_$filename");
|
||||
file_put_contents($tempFilePath, $disk->get($filePath));
|
||||
|
||||
Log::info('Temporary file created', ['temp_path' => $tempFilePath]);
|
||||
return $tempFilePath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Proses file CSV
|
||||
*
|
||||
* @param string $tempFilePath Path file temporary
|
||||
* @param string $filePath Path file asli
|
||||
* @return void
|
||||
*/
|
||||
private function processFile(string $tempFilePath, string $filePath): void
|
||||
{
|
||||
$handle = fopen($tempFilePath, "r");
|
||||
if ($handle === false) {
|
||||
Log::error("Unable to open file: $filePath");
|
||||
return;
|
||||
}
|
||||
|
||||
$headers = (new StmtEntryDetail())->getFillable();
|
||||
$rowCount = 0;
|
||||
$chunkCount = 0;
|
||||
|
||||
Log::info('Memulai proses file', [
|
||||
'file_path' => $filePath,
|
||||
'headers_count' => count($headers)
|
||||
]);
|
||||
|
||||
while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) {
|
||||
$rowCount++;
|
||||
$this->processRow($row, $headers, $rowCount, $filePath);
|
||||
|
||||
// Process in chunks to avoid memory issues
|
||||
if (count($this->entryBatch) >= self::CHUNK_SIZE) {
|
||||
$this->saveBatch();
|
||||
$chunkCount++;
|
||||
Log::info("Processed chunk $chunkCount ({$this->processedCount} records so far)");
|
||||
}
|
||||
}
|
||||
|
||||
// Process any remaining records
|
||||
if (!empty($this->entryBatch)) {
|
||||
$this->saveBatch();
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors.");
|
||||
}
|
||||
|
||||
/**
|
||||
* Proses setiap baris data
|
||||
*
|
||||
* @param array $row Data baris
|
||||
* @param array $headers Header kolom
|
||||
* @param int $rowCount Nomor baris
|
||||
* @param string $filePath Path file
|
||||
* @return void
|
||||
*/
|
||||
private function processRow(array $row, array $headers, int $rowCount, string $filePath): void
|
||||
{
|
||||
if (count($headers) !== count($row)) {
|
||||
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " .
|
||||
count($headers) . ", Got: " . count($row));
|
||||
return;
|
||||
}
|
||||
|
||||
$data = array_combine($headers, $row);
|
||||
$this->cleanTransReference($data);
|
||||
$this->addToBatch($data, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Bersihkan trans_reference dari karakter tidak diinginkan
|
||||
*
|
||||
* @param array $data Data yang akan dibersihkan
|
||||
* @return void
|
||||
*/
|
||||
private function cleanTransReference(array &$data): void
|
||||
{
|
||||
if (isset($data['trans_reference'])) {
|
||||
// Clean trans_reference from \\BNK if present
|
||||
$data['trans_reference'] = preg_replace('/\\\\.*$/', '', $data['trans_reference']);
|
||||
Log::debug('Trans reference cleaned', ['original' => $data['trans_reference']]);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Tambahkan record ke batch untuk proses bulk insert
|
||||
*
|
||||
* @param array $data Data record
|
||||
* @param int $rowCount Nomor baris
|
||||
* @param string $filePath Path file
|
||||
* @return void
|
||||
*/
|
||||
private function addToBatch(array $data, int $rowCount, string $filePath): void
|
||||
{
|
||||
try {
|
||||
if (isset($data['stmt_entry_id']) && $data['stmt_entry_id'] !== 'stmt_entry_id') {
|
||||
// Add timestamp fields
|
||||
$now = now();
|
||||
$data['created_at'] = $now;
|
||||
$data['updated_at'] = $now;
|
||||
|
||||
// Add to entry batch
|
||||
$this->entryBatch[] = $data;
|
||||
$this->processedCount++;
|
||||
|
||||
Log::debug('Record added to batch', ['row' => $rowCount, 'stmt_entry_id' => $data['stmt_entry_id']]);
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
$this->errorCount++;
|
||||
Log::error("Error processing Statement Entry Detail at row $rowCount in $filePath: " . $e->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Simpan batch data ke database menggunakan updateOrCreate
|
||||
* untuk menghindari error unique constraint
|
||||
*
|
||||
* @return void
|
||||
* @throws Exception
|
||||
*/
|
||||
private function saveBatch(): void
|
||||
{
|
||||
Log::info('Memulai proses saveBatch dengan updateOrCreate');
|
||||
|
||||
DB::beginTransaction();
|
||||
|
||||
try {
|
||||
if (!empty($this->entryBatch)) {
|
||||
$totalProcessed = 0;
|
||||
|
||||
// Process each entry data directly
|
||||
foreach ($this->entryBatch as $entryData) {
|
||||
// Validasi bahwa entryData adalah array dan memiliki stmt_entry_id
|
||||
if (is_array($entryData) && isset($entryData['stmt_entry_id'])) {
|
||||
// Gunakan updateOrCreate untuk menghindari duplicate key error
|
||||
StmtEntryDetail::updateOrCreate(
|
||||
[
|
||||
'stmt_entry_id' => $entryData['stmt_entry_id']
|
||||
],
|
||||
$entryData
|
||||
);
|
||||
|
||||
$totalProcessed++;
|
||||
} else {
|
||||
Log::warning('Invalid entry data structure', ['data' => $entryData]);
|
||||
$this->errorCount++;
|
||||
}
|
||||
}
|
||||
|
||||
DB::commit();
|
||||
|
||||
Log::info("Berhasil memproses {$totalProcessed} record dengan updateOrCreate");
|
||||
|
||||
// Reset entry batch after successful processing
|
||||
$this->entryBatch = [];
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
DB::rollback();
|
||||
|
||||
Log::error("Error in saveBatch: " . $e->getMessage() . "\n" . $e->getTraceAsString());
|
||||
$this->errorCount += count($this->entryBatch);
|
||||
|
||||
// Reset batch even if there's an error to prevent reprocessing the same failed records
|
||||
$this->entryBatch = [];
|
||||
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Bersihkan file temporary
|
||||
*
|
||||
* @param string $tempFilePath Path file temporary
|
||||
* @return void
|
||||
*/
|
||||
private function cleanup(string $tempFilePath): void
|
||||
{
|
||||
if (file_exists($tempFilePath)) {
|
||||
unlink($tempFilePath);
|
||||
Log::info('Temporary file cleaned up', ['temp_path' => $tempFilePath]);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Log penyelesaian job
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
private function logJobCompletion(): void
|
||||
{
|
||||
Log::info("Statement Entry Detail data processing completed. " .
|
||||
"Total processed: {$this->processedCount}, Total errors: {$this->errorCount}");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user