✨ feat(slik): implementasi sistem import SLIK dengan optimasi memory & timeout handling
- Menambahkan `SlikController.php` dengan method CRUD dan import data SLIK, termasuk logging detail & error handling - Menambahkan `SlikImport.php` dengan Laravel Excel (ToCollection, WithChunkReading, WithBatchInserts, dll.) - Optimasi memory dengan chunk processing (50 baris/chunk) dan batch insert (50 record/batch) - Penanganan timeout menggunakan `set_time_limit` & memory limit configurable via config - Implementasi queue processing untuk file besar (>5MB) dengan progress tracking - Validasi file upload & data baris, skip header dari baris ke-5, serta rollback jika error - Garbage collection otomatis setiap 25 baris, unset variabel tidak terpakai, dan logging usage memory - Error handling komprehensif dengan try-catch, rollback transaksi, hapus file temp, dan logging stack trace - Semua parameter (batch size, chunk size, memory limit, timeout, GC, queue threshold) configurable via config - Diuji pada file besar (>50MB), memory stabil, timeout handling berfungsi, rollback aman, dan progress tracking valid - Catatan: pastikan queue worker berjalan, monitor log progress, sesuaikan config server, dan backup DB sebelum import
This commit is contained in:
Daeng Deni Mardaeni
415
app/Imports/SlikImport.php
Normal file
415
app/Imports/SlikImport.php
Normal file
@@ -0,0 +1,415 @@
|
||||
<?php
|
||||
|
||||
namespace Modules\Lpj\Imports;
|
||||
|
||||
use Modules\Lpj\Models\Slik;
|
||||
use Illuminate\Support\Collection;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Maatwebsite\Excel\Concerns\ToCollection;
|
||||
use Maatwebsite\Excel\Concerns\WithStartRow;
|
||||
use Maatwebsite\Excel\Concerns\WithChunkReading;
|
||||
use Maatwebsite\Excel\Concerns\WithBatchInserts;
|
||||
use Maatwebsite\Excel\Concerns\WithCustomCsvSettings;
|
||||
|
||||
|
||||
/**
|
||||
* Class SlikImport
|
||||
*
|
||||
* Handle import data Excel untuk modul Slik
|
||||
* Menggunakan Laravel Excel package untuk membaca file Excel
|
||||
* dengan optimasi memory dan chunk processing
|
||||
*
|
||||
* @package Modules\Lpj\app\Imports
|
||||
*/
|
||||
class SlikImport implements ToCollection, WithStartRow, WithBatchInserts, WithChunkReading, WithCustomCsvSettings
|
||||
{
|
||||
/**
|
||||
* Mulai membaca dari baris ke-5 (skip header)
|
||||
*
|
||||
* @return int
|
||||
*/
|
||||
public function startRow(): int
|
||||
{
|
||||
return 5;
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch size untuk insert data dari konfigurasi
|
||||
*
|
||||
* @return int
|
||||
*/
|
||||
public function batchSize(): int
|
||||
{
|
||||
return config('import.slik.batch_size', 50);
|
||||
}
|
||||
|
||||
/**
|
||||
* Chunk size untuk membaca file dari konfigurasi
|
||||
*
|
||||
* @return int
|
||||
*/
|
||||
public function chunkSize(): int
|
||||
{
|
||||
return config('import.slik.chunk_size', 50);
|
||||
}
|
||||
|
||||
/**
|
||||
* Custom CSV settings untuk optimasi pembacaan file
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getCsvSettings(): array
|
||||
{
|
||||
return [
|
||||
'input_encoding' => 'UTF-8',
|
||||
'delimiter' => ',',
|
||||
'enclosure' => '"',
|
||||
'escape_character' => '\\',
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Process collection data dari Excel dengan optimasi memory
|
||||
*
|
||||
* @param Collection $collection
|
||||
* @return void
|
||||
*/
|
||||
public function collection(Collection $collection)
|
||||
{
|
||||
// Set memory limit dari konfigurasi
|
||||
$memoryLimit = config('import.slik.memory_limit', 1024);
|
||||
$currentMemoryLimit = ini_get('memory_limit');
|
||||
|
||||
if ($currentMemoryLimit !== '-1' && $this->convertToBytes($currentMemoryLimit) < $memoryLimit * 1024 * 1024) {
|
||||
ini_set('memory_limit', $memoryLimit . 'M');
|
||||
}
|
||||
|
||||
// Set timeout handler
|
||||
$timeout = config('import.slik.timeout', 1800);
|
||||
set_time_limit($timeout);
|
||||
|
||||
// Force garbage collection sebelum memulai
|
||||
if (config('import.slik.enable_gc', true)) {
|
||||
gc_collect_cycles();
|
||||
}
|
||||
|
||||
Log::info('SlikImport: Memulai import data', [
|
||||
'total_rows' => $collection->count(),
|
||||
'memory_usage' => memory_get_usage(true),
|
||||
'memory_peak' => memory_get_peak_usage(true),
|
||||
'memory_limit' => ini_get('memory_limit'),
|
||||
'php_version' => PHP_VERSION,
|
||||
'memory_limit_before' => $currentMemoryLimit,
|
||||
'config' => [
|
||||
'memory_limit' => $memoryLimit,
|
||||
'chunk_size' => $this->chunkSize(),
|
||||
'batch_size' => $this->batchSize()
|
||||
]
|
||||
]);
|
||||
|
||||
DB::beginTransaction();
|
||||
|
||||
try {
|
||||
$processedRows = 0;
|
||||
$skippedRows = 0;
|
||||
$errorRows = 0;
|
||||
$totalRows = $collection->count();
|
||||
|
||||
foreach ($collection as $index => $row) {
|
||||
// Log progress setiap 25 baris untuk chunk lebih kecil
|
||||
if ($index % 25 === 0) {
|
||||
Log::info('SlikImport: Processing chunk', [
|
||||
'current_row' => $index + 5,
|
||||
'progress' => round(($index / max($totalRows, 1)) * 100, 2) . '%',
|
||||
'processed' => $processedRows,
|
||||
'skipped' => $skippedRows,
|
||||
'errors' => $errorRows,
|
||||
'memory_usage' => memory_get_usage(true),
|
||||
'memory_peak' => memory_get_peak_usage(true),
|
||||
'memory_diff' => memory_get_peak_usage(true) - memory_get_usage(true)
|
||||
]);
|
||||
}
|
||||
|
||||
// Skip baris kosong
|
||||
if ($this->isEmptyRow($row)) {
|
||||
$skippedRows++;
|
||||
Log::debug('SlikImport: Skipping empty row', ['row_number' => $index + 5]);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Validasi data
|
||||
if (!$this->validateRow($row)) {
|
||||
$errorRows++;
|
||||
Log::warning('SlikImport: Invalid row data', [
|
||||
'row_number' => $index + 5,
|
||||
'data' => $row->toArray()
|
||||
]);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
// Map data dari Excel ke model
|
||||
$slikData = $this->mapRowToSlik($row);
|
||||
|
||||
// Update atau create berdasarkan no_rekening dan cif
|
||||
$slik = Slik::updateOrCreate(
|
||||
[
|
||||
'no_rekening' => $slikData['no_rekening'],
|
||||
'cif' => $slikData['cif']
|
||||
],
|
||||
$slikData
|
||||
);
|
||||
|
||||
$processedRows++;
|
||||
|
||||
// Log detail untuk baris pertama sebagai sample
|
||||
if ($index === 0) {
|
||||
Log::info('SlikImport: Sample data processed', [
|
||||
'slik_id' => $slik->id,
|
||||
'no_rekening' => $slik->no_rekening,
|
||||
'cif' => $slik->cif,
|
||||
'was_recently_created' => $slik->wasRecentlyCreated
|
||||
]);
|
||||
}
|
||||
|
||||
// Force garbage collection setiap 25 baris untuk mengurangi memory
|
||||
if (config('import.slik.enable_gc', true) && $index > 0 && $index % 25 === 0) {
|
||||
gc_collect_cycles();
|
||||
}
|
||||
|
||||
// Unset data yang sudah tidak digunakan untuk mengurangi memory
|
||||
if ($index > 0 && $index % 25 === 0) {
|
||||
unset($slikData, $slik);
|
||||
}
|
||||
|
||||
// Reset collection internal untuk mengurangi memory
|
||||
if ($index > 0 && $index % 100 === 0) {
|
||||
$collection = collect($collection->slice($index + 1)->values());
|
||||
}
|
||||
|
||||
} catch (\Exception $e) {
|
||||
$errorRows++;
|
||||
Log::error('SlikImport: Error processing row', [
|
||||
'row_number' => $index + 5,
|
||||
'error' => $e->getMessage(),
|
||||
'data' => $row->toArray(),
|
||||
'memory_usage' => memory_get_usage(true)
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
DB::commit();
|
||||
|
||||
// Force garbage collection setelah selesai
|
||||
if (config('import.slik.enable_gc', true)) {
|
||||
gc_collect_cycles();
|
||||
}
|
||||
|
||||
// Cleanup variables
|
||||
unset($collection);
|
||||
|
||||
Log::info('SlikImport: Import berhasil diselesaikan', [
|
||||
'total_rows' => $totalRows,
|
||||
'processed_rows' => $processedRows,
|
||||
'skipped_rows' => $skippedRows,
|
||||
'error_rows' => $errorRows,
|
||||
'final_memory_usage' => memory_get_usage(true),
|
||||
'peak_memory_usage' => memory_get_peak_usage(true),
|
||||
'memory_saved' => memory_get_peak_usage(true) - memory_get_usage(true),
|
||||
'memory_efficiency' => ($processedRows > 0) ? round(memory_get_peak_usage(true) / $processedRows, 2) : 0
|
||||
]);
|
||||
|
||||
} catch (\Exception $e) {
|
||||
DB::rollBack();
|
||||
|
||||
// Force garbage collection jika error
|
||||
if (config('import.slik.enable_gc', true)) {
|
||||
gc_collect_cycles();
|
||||
}
|
||||
|
||||
$errorType = 'general';
|
||||
if (str_contains(strtolower($e->getMessage()), 'memory')) {
|
||||
$errorType = 'memory';
|
||||
} elseif (str_contains(strtolower($e->getMessage()), 'timeout') || str_contains(strtolower($e->getMessage()), 'maximum execution time')) {
|
||||
$errorType = 'timeout';
|
||||
}
|
||||
|
||||
Log::error('SlikImport: Error during import', [
|
||||
'error' => $e->getMessage(),
|
||||
'error_type' => $errorType,
|
||||
'exception_type' => get_class($e),
|
||||
'trace' => $e->getTraceAsString(),
|
||||
'file' => $e->getFile(),
|
||||
'line' => $e->getLine(),
|
||||
'memory_usage' => memory_get_usage(true),
|
||||
'memory_peak' => memory_get_peak_usage(true),
|
||||
'memory_limit' => ini_get('memory_limit'),
|
||||
'timeout_limit' => ini_get('max_execution_time'),
|
||||
'is_memory_error' => str_contains(strtolower($e->getMessage()), 'memory'),
|
||||
'is_timeout_error' => str_contains(strtolower($e->getMessage()), 'timeout') || str_contains(strtolower($e->getMessage()), 'maximum execution time')
|
||||
]);
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert memory limit string ke bytes
|
||||
*
|
||||
* @param string $memoryLimit
|
||||
* @return int
|
||||
*/
|
||||
private function convertToBytes(string $memoryLimit): int
|
||||
{
|
||||
$memoryLimit = trim($memoryLimit);
|
||||
$lastChar = strtolower(substr($memoryLimit, -1));
|
||||
$number = (int) substr($memoryLimit, 0, -1);
|
||||
|
||||
switch ($lastChar) {
|
||||
case 'g':
|
||||
return $number * 1024 * 1024 * 1024;
|
||||
case 'm':
|
||||
return $number * 1024 * 1024;
|
||||
case 'k':
|
||||
return $number * 1024;
|
||||
default:
|
||||
return (int) $memoryLimit;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cek apakah baris kosong
|
||||
*
|
||||
* @param Collection $row
|
||||
* @return bool
|
||||
*/
|
||||
private function isEmptyRow(Collection $row): bool
|
||||
{
|
||||
return $row->filter(function ($value) {
|
||||
return !empty(trim($value));
|
||||
})->isEmpty();
|
||||
}
|
||||
|
||||
/**
|
||||
* Validasi data baris
|
||||
*
|
||||
* @param Collection $row
|
||||
* @return bool
|
||||
*/
|
||||
private function validateRow(Collection $row): bool
|
||||
{
|
||||
// Validasi minimal: sandi_bank, no_rekening, dan cif harus ada
|
||||
return !empty(trim($row[0])) && // sandi_bank
|
||||
!empty(trim($row[5])) && // no_rekening
|
||||
!empty(trim($row[6])); // cif
|
||||
}
|
||||
|
||||
/**
|
||||
* Map data dari baris Excel ke array untuk model Slik
|
||||
*
|
||||
* @param Collection $row
|
||||
* @return array
|
||||
*/
|
||||
private function mapRowToSlik(Collection $row): array
|
||||
{
|
||||
return [
|
||||
'sandi_bank' => trim($row[0]) ?: null,
|
||||
'tahun' => $this->parseInteger($row[1]),
|
||||
'bulan' => $this->parseInteger($row[2]),
|
||||
'flag_detail' => trim($row[3]) ?: null,
|
||||
'kode_register_agunan' => trim($row[4]) ?: null,
|
||||
'no_rekening' => trim($row[5]) ?: null,
|
||||
'cif' => trim($row[6]) ?: null,
|
||||
'kolektibilitas' => trim($row[7]) ?: null,
|
||||
'fasilitas' => trim($row[8]) ?: null,
|
||||
'jenis_segmen_fasilitas' => trim($row[9]) ?: null,
|
||||
'status_agunan' => trim($row[10]) ?: null,
|
||||
'jenis_agunan' => trim($row[11]) ?: null,
|
||||
'peringkat_agunan' => trim($row[12]) ?: null,
|
||||
'lembaga_pemeringkat' => trim($row[13]) ?: null,
|
||||
'jenis_pengikatan' => trim($row[14]) ?: null,
|
||||
'tanggal_pengikatan' => $this->parseDate($row[15]),
|
||||
'nama_pemilik_agunan' => trim($row[16]) ?: null,
|
||||
'bukti_kepemilikan' => trim($row[17]) ?: null,
|
||||
'alamat_agunan' => trim($row[18]) ?: null,
|
||||
'lokasi_agunan' => trim($row[19]) ?: null,
|
||||
'nilai_agunan' => $this->parseDecimal($row[20]),
|
||||
'nilai_agunan_menurut_ljk' => $this->parseDecimal($row[21]),
|
||||
'tanggal_penilaian_ljk' => $this->parseDate($row[22]),
|
||||
'nilai_agunan_penilai_independen' => $this->parseDecimal($row[23]),
|
||||
'nama_penilai_independen' => trim($row[24]) ?: null,
|
||||
'tanggal_penilaian_penilai_independen' => $this->parseDate($row[25]),
|
||||
'jumlah_hari_tunggakan' => $this->parseInteger($row[26]),
|
||||
'status_paripasu' => trim($row[27]) ?: null,
|
||||
'prosentase_paripasu' => $this->parseDecimal($row[28]),
|
||||
'status_kredit_join' => trim($row[29]) ?: null,
|
||||
'diasuransikan' => trim($row[30]) ?: null,
|
||||
'keterangan' => trim($row[31]) ?: null,
|
||||
'kantor_cabang' => trim($row[32]) ?: null,
|
||||
'operasi_data' => trim($row[33]) ?: null,
|
||||
'kode_cabang' => trim($row[34]) ?: null,
|
||||
'nama_debitur' => trim($row[35]) ?: null,
|
||||
'nama_cabang' => trim($row[36]) ?: null,
|
||||
'flag' => trim($row[37]) ?: null,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse integer value
|
||||
*
|
||||
* @param mixed $value
|
||||
* @return int|null
|
||||
*/
|
||||
private function parseInteger($value): ?int
|
||||
{
|
||||
if (empty(trim($value))) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (int) $value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse decimal value
|
||||
*
|
||||
* @param mixed $value
|
||||
* @return float|null
|
||||
*/
|
||||
private function parseDecimal($value): ?float
|
||||
{
|
||||
if (empty(trim($value))) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Remove currency formatting
|
||||
$cleaned = str_replace([',', '.'], ['', '.'], $value);
|
||||
$cleaned = preg_replace('/[^0-9.]/', '', $cleaned);
|
||||
|
||||
return (float) $cleaned;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse date value
|
||||
*
|
||||
* @param mixed $value
|
||||
* @return string|null
|
||||
*/
|
||||
private function parseDate($value): ?string
|
||||
{
|
||||
if (empty(trim($value))) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
// Try to parse various date formats
|
||||
$date = \Carbon\Carbon::parse($value);
|
||||
return $date->format('Y-m-d');
|
||||
} catch (\Exception $e) {
|
||||
Log::warning('SlikImport: Invalid date format', [
|
||||
'value' => $value,
|
||||
'error' => $e->getMessage()
|
||||
]);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user