20833213b1
- Menambahkan `SlikController.php` dengan method CRUD dan import data SLIK, termasuk logging detail & error handling - Menambahkan `SlikImport.php` dengan Laravel Excel (ToCollection, WithChunkReading, WithBatchInserts, dll.) - Optimasi memory dengan chunk processing (50 baris/chunk) dan batch insert (50 record/batch) - Penanganan timeout menggunakan `set_time_limit` & memory limit configurable via config - Implementasi queue processing untuk file besar (>5MB) dengan progress tracking - Validasi file upload & data baris, skip header dari baris ke-5, serta rollback jika error - Garbage collection otomatis setiap 25 baris, unset variabel tidak terpakai, dan logging usage memory - Error handling komprehensif dengan try-catch, rollback transaksi, hapus file temp, dan logging stack trace - Semua parameter (batch size, chunk size, memory limit, timeout, GC, queue threshold) configurable via config - Diuji pada file besar (>50MB), memory stabil, timeout handling berfungsi, rollback aman, dan progress tracking valid - Catatan: pastikan queue worker berjalan, monitor log progress, sesuaikan config server, dan backup DB sebelum import
416 lines
14 KiB
PHP
416 lines
14 KiB
PHP
<?php
|
|
|
|
namespace Modules\Lpj\Imports;
|
|
|
|
use Modules\Lpj\Models\Slik;
|
|
use Illuminate\Support\Collection;
|
|
use Illuminate\Support\Facades\DB;
|
|
use Illuminate\Support\Facades\Log;
|
|
use Maatwebsite\Excel\Concerns\ToCollection;
|
|
use Maatwebsite\Excel\Concerns\WithStartRow;
|
|
use Maatwebsite\Excel\Concerns\WithChunkReading;
|
|
use Maatwebsite\Excel\Concerns\WithBatchInserts;
|
|
use Maatwebsite\Excel\Concerns\WithCustomCsvSettings;
|
|
|
|
|
|
/**
|
|
* Class SlikImport
|
|
*
|
|
* Handle import data Excel untuk modul Slik
|
|
* Menggunakan Laravel Excel package untuk membaca file Excel
|
|
* dengan optimasi memory dan chunk processing
|
|
*
|
|
* @package Modules\Lpj\app\Imports
|
|
*/
|
|
class SlikImport implements ToCollection, WithStartRow, WithBatchInserts, WithChunkReading, WithCustomCsvSettings
|
|
{
|
|
/**
|
|
* Mulai membaca dari baris ke-5 (skip header)
|
|
*
|
|
* @return int
|
|
*/
|
|
public function startRow(): int
|
|
{
|
|
return 5;
|
|
}
|
|
|
|
/**
|
|
* Batch size untuk insert data dari konfigurasi
|
|
*
|
|
* @return int
|
|
*/
|
|
public function batchSize(): int
|
|
{
|
|
return config('import.slik.batch_size', 50);
|
|
}
|
|
|
|
/**
|
|
* Chunk size untuk membaca file dari konfigurasi
|
|
*
|
|
* @return int
|
|
*/
|
|
public function chunkSize(): int
|
|
{
|
|
return config('import.slik.chunk_size', 50);
|
|
}
|
|
|
|
/**
|
|
* Custom CSV settings untuk optimasi pembacaan file
|
|
*
|
|
* @return array
|
|
*/
|
|
public function getCsvSettings(): array
|
|
{
|
|
return [
|
|
'input_encoding' => 'UTF-8',
|
|
'delimiter' => ',',
|
|
'enclosure' => '"',
|
|
'escape_character' => '\\',
|
|
];
|
|
}
|
|
|
|
/**
|
|
* Process collection data dari Excel dengan optimasi memory
|
|
*
|
|
* @param Collection $collection
|
|
* @return void
|
|
*/
|
|
public function collection(Collection $collection)
|
|
{
|
|
// Set memory limit dari konfigurasi
|
|
$memoryLimit = config('import.slik.memory_limit', 1024);
|
|
$currentMemoryLimit = ini_get('memory_limit');
|
|
|
|
if ($currentMemoryLimit !== '-1' && $this->convertToBytes($currentMemoryLimit) < $memoryLimit * 1024 * 1024) {
|
|
ini_set('memory_limit', $memoryLimit . 'M');
|
|
}
|
|
|
|
// Set timeout handler
|
|
$timeout = config('import.slik.timeout', 1800);
|
|
set_time_limit($timeout);
|
|
|
|
// Force garbage collection sebelum memulai
|
|
if (config('import.slik.enable_gc', true)) {
|
|
gc_collect_cycles();
|
|
}
|
|
|
|
Log::info('SlikImport: Memulai import data', [
|
|
'total_rows' => $collection->count(),
|
|
'memory_usage' => memory_get_usage(true),
|
|
'memory_peak' => memory_get_peak_usage(true),
|
|
'memory_limit' => ini_get('memory_limit'),
|
|
'php_version' => PHP_VERSION,
|
|
'memory_limit_before' => $currentMemoryLimit,
|
|
'config' => [
|
|
'memory_limit' => $memoryLimit,
|
|
'chunk_size' => $this->chunkSize(),
|
|
'batch_size' => $this->batchSize()
|
|
]
|
|
]);
|
|
|
|
DB::beginTransaction();
|
|
|
|
try {
|
|
$processedRows = 0;
|
|
$skippedRows = 0;
|
|
$errorRows = 0;
|
|
$totalRows = $collection->count();
|
|
|
|
foreach ($collection as $index => $row) {
|
|
// Log progress setiap 25 baris untuk chunk lebih kecil
|
|
if ($index % 25 === 0) {
|
|
Log::info('SlikImport: Processing chunk', [
|
|
'current_row' => $index + 5,
|
|
'progress' => round(($index / max($totalRows, 1)) * 100, 2) . '%',
|
|
'processed' => $processedRows,
|
|
'skipped' => $skippedRows,
|
|
'errors' => $errorRows,
|
|
'memory_usage' => memory_get_usage(true),
|
|
'memory_peak' => memory_get_peak_usage(true),
|
|
'memory_diff' => memory_get_peak_usage(true) - memory_get_usage(true)
|
|
]);
|
|
}
|
|
|
|
// Skip baris kosong
|
|
if ($this->isEmptyRow($row)) {
|
|
$skippedRows++;
|
|
Log::debug('SlikImport: Skipping empty row', ['row_number' => $index + 5]);
|
|
continue;
|
|
}
|
|
|
|
// Validasi data
|
|
if (!$this->validateRow($row)) {
|
|
$errorRows++;
|
|
Log::warning('SlikImport: Invalid row data', [
|
|
'row_number' => $index + 5,
|
|
'data' => $row->toArray()
|
|
]);
|
|
continue;
|
|
}
|
|
|
|
try {
|
|
// Map data dari Excel ke model
|
|
$slikData = $this->mapRowToSlik($row);
|
|
|
|
// Update atau create berdasarkan no_rekening dan cif
|
|
$slik = Slik::updateOrCreate(
|
|
[
|
|
'no_rekening' => $slikData['no_rekening'],
|
|
'cif' => $slikData['cif']
|
|
],
|
|
$slikData
|
|
);
|
|
|
|
$processedRows++;
|
|
|
|
// Log detail untuk baris pertama sebagai sample
|
|
if ($index === 0) {
|
|
Log::info('SlikImport: Sample data processed', [
|
|
'slik_id' => $slik->id,
|
|
'no_rekening' => $slik->no_rekening,
|
|
'cif' => $slik->cif,
|
|
'was_recently_created' => $slik->wasRecentlyCreated
|
|
]);
|
|
}
|
|
|
|
// Force garbage collection setiap 25 baris untuk mengurangi memory
|
|
if (config('import.slik.enable_gc', true) && $index > 0 && $index % 25 === 0) {
|
|
gc_collect_cycles();
|
|
}
|
|
|
|
// Unset data yang sudah tidak digunakan untuk mengurangi memory
|
|
if ($index > 0 && $index % 25 === 0) {
|
|
unset($slikData, $slik);
|
|
}
|
|
|
|
// Reset collection internal untuk mengurangi memory
|
|
if ($index > 0 && $index % 100 === 0) {
|
|
$collection = collect($collection->slice($index + 1)->values());
|
|
}
|
|
|
|
} catch (\Exception $e) {
|
|
$errorRows++;
|
|
Log::error('SlikImport: Error processing row', [
|
|
'row_number' => $index + 5,
|
|
'error' => $e->getMessage(),
|
|
'data' => $row->toArray(),
|
|
'memory_usage' => memory_get_usage(true)
|
|
]);
|
|
}
|
|
}
|
|
|
|
DB::commit();
|
|
|
|
// Force garbage collection setelah selesai
|
|
if (config('import.slik.enable_gc', true)) {
|
|
gc_collect_cycles();
|
|
}
|
|
|
|
// Cleanup variables
|
|
unset($collection);
|
|
|
|
Log::info('SlikImport: Import berhasil diselesaikan', [
|
|
'total_rows' => $totalRows,
|
|
'processed_rows' => $processedRows,
|
|
'skipped_rows' => $skippedRows,
|
|
'error_rows' => $errorRows,
|
|
'final_memory_usage' => memory_get_usage(true),
|
|
'peak_memory_usage' => memory_get_peak_usage(true),
|
|
'memory_saved' => memory_get_peak_usage(true) - memory_get_usage(true),
|
|
'memory_efficiency' => ($processedRows > 0) ? round(memory_get_peak_usage(true) / $processedRows, 2) : 0
|
|
]);
|
|
|
|
} catch (\Exception $e) {
|
|
DB::rollBack();
|
|
|
|
// Force garbage collection jika error
|
|
if (config('import.slik.enable_gc', true)) {
|
|
gc_collect_cycles();
|
|
}
|
|
|
|
$errorType = 'general';
|
|
if (str_contains(strtolower($e->getMessage()), 'memory')) {
|
|
$errorType = 'memory';
|
|
} elseif (str_contains(strtolower($e->getMessage()), 'timeout') || str_contains(strtolower($e->getMessage()), 'maximum execution time')) {
|
|
$errorType = 'timeout';
|
|
}
|
|
|
|
Log::error('SlikImport: Error during import', [
|
|
'error' => $e->getMessage(),
|
|
'error_type' => $errorType,
|
|
'exception_type' => get_class($e),
|
|
'trace' => $e->getTraceAsString(),
|
|
'file' => $e->getFile(),
|
|
'line' => $e->getLine(),
|
|
'memory_usage' => memory_get_usage(true),
|
|
'memory_peak' => memory_get_peak_usage(true),
|
|
'memory_limit' => ini_get('memory_limit'),
|
|
'timeout_limit' => ini_get('max_execution_time'),
|
|
'is_memory_error' => str_contains(strtolower($e->getMessage()), 'memory'),
|
|
'is_timeout_error' => str_contains(strtolower($e->getMessage()), 'timeout') || str_contains(strtolower($e->getMessage()), 'maximum execution time')
|
|
]);
|
|
throw $e;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Convert memory limit string ke bytes
|
|
*
|
|
* @param string $memoryLimit
|
|
* @return int
|
|
*/
|
|
private function convertToBytes(string $memoryLimit): int
|
|
{
|
|
$memoryLimit = trim($memoryLimit);
|
|
$lastChar = strtolower(substr($memoryLimit, -1));
|
|
$number = (int) substr($memoryLimit, 0, -1);
|
|
|
|
switch ($lastChar) {
|
|
case 'g':
|
|
return $number * 1024 * 1024 * 1024;
|
|
case 'm':
|
|
return $number * 1024 * 1024;
|
|
case 'k':
|
|
return $number * 1024;
|
|
default:
|
|
return (int) $memoryLimit;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Cek apakah baris kosong
|
|
*
|
|
* @param Collection $row
|
|
* @return bool
|
|
*/
|
|
private function isEmptyRow(Collection $row): bool
|
|
{
|
|
return $row->filter(function ($value) {
|
|
return !empty(trim($value));
|
|
})->isEmpty();
|
|
}
|
|
|
|
/**
|
|
* Validasi data baris
|
|
*
|
|
* @param Collection $row
|
|
* @return bool
|
|
*/
|
|
private function validateRow(Collection $row): bool
|
|
{
|
|
// Validasi minimal: sandi_bank, no_rekening, dan cif harus ada
|
|
return !empty(trim($row[0])) && // sandi_bank
|
|
!empty(trim($row[5])) && // no_rekening
|
|
!empty(trim($row[6])); // cif
|
|
}
|
|
|
|
/**
|
|
* Map data dari baris Excel ke array untuk model Slik
|
|
*
|
|
* @param Collection $row
|
|
* @return array
|
|
*/
|
|
private function mapRowToSlik(Collection $row): array
|
|
{
|
|
return [
|
|
'sandi_bank' => trim($row[0]) ?: null,
|
|
'tahun' => $this->parseInteger($row[1]),
|
|
'bulan' => $this->parseInteger($row[2]),
|
|
'flag_detail' => trim($row[3]) ?: null,
|
|
'kode_register_agunan' => trim($row[4]) ?: null,
|
|
'no_rekening' => trim($row[5]) ?: null,
|
|
'cif' => trim($row[6]) ?: null,
|
|
'kolektibilitas' => trim($row[7]) ?: null,
|
|
'fasilitas' => trim($row[8]) ?: null,
|
|
'jenis_segmen_fasilitas' => trim($row[9]) ?: null,
|
|
'status_agunan' => trim($row[10]) ?: null,
|
|
'jenis_agunan' => trim($row[11]) ?: null,
|
|
'peringkat_agunan' => trim($row[12]) ?: null,
|
|
'lembaga_pemeringkat' => trim($row[13]) ?: null,
|
|
'jenis_pengikatan' => trim($row[14]) ?: null,
|
|
'tanggal_pengikatan' => $this->parseDate($row[15]),
|
|
'nama_pemilik_agunan' => trim($row[16]) ?: null,
|
|
'bukti_kepemilikan' => trim($row[17]) ?: null,
|
|
'alamat_agunan' => trim($row[18]) ?: null,
|
|
'lokasi_agunan' => trim($row[19]) ?: null,
|
|
'nilai_agunan' => $this->parseDecimal($row[20]),
|
|
'nilai_agunan_menurut_ljk' => $this->parseDecimal($row[21]),
|
|
'tanggal_penilaian_ljk' => $this->parseDate($row[22]),
|
|
'nilai_agunan_penilai_independen' => $this->parseDecimal($row[23]),
|
|
'nama_penilai_independen' => trim($row[24]) ?: null,
|
|
'tanggal_penilaian_penilai_independen' => $this->parseDate($row[25]),
|
|
'jumlah_hari_tunggakan' => $this->parseInteger($row[26]),
|
|
'status_paripasu' => trim($row[27]) ?: null,
|
|
'prosentase_paripasu' => $this->parseDecimal($row[28]),
|
|
'status_kredit_join' => trim($row[29]) ?: null,
|
|
'diasuransikan' => trim($row[30]) ?: null,
|
|
'keterangan' => trim($row[31]) ?: null,
|
|
'kantor_cabang' => trim($row[32]) ?: null,
|
|
'operasi_data' => trim($row[33]) ?: null,
|
|
'kode_cabang' => trim($row[34]) ?: null,
|
|
'nama_debitur' => trim($row[35]) ?: null,
|
|
'nama_cabang' => trim($row[36]) ?: null,
|
|
'flag' => trim($row[37]) ?: null,
|
|
];
|
|
}
|
|
|
|
/**
|
|
* Parse integer value
|
|
*
|
|
* @param mixed $value
|
|
* @return int|null
|
|
*/
|
|
private function parseInteger($value): ?int
|
|
{
|
|
if (empty(trim($value))) {
|
|
return null;
|
|
}
|
|
|
|
return (int) $value;
|
|
}
|
|
|
|
/**
|
|
* Parse decimal value
|
|
*
|
|
* @param mixed $value
|
|
* @return float|null
|
|
*/
|
|
private function parseDecimal($value): ?float
|
|
{
|
|
if (empty(trim($value))) {
|
|
return null;
|
|
}
|
|
|
|
// Remove currency formatting
|
|
$cleaned = str_replace([',', '.'], ['', '.'], $value);
|
|
$cleaned = preg_replace('/[^0-9.]/', '', $cleaned);
|
|
|
|
return (float) $cleaned;
|
|
}
|
|
|
|
/**
|
|
* Parse date value
|
|
*
|
|
* @param mixed $value
|
|
* @return string|null
|
|
*/
|
|
private function parseDate($value): ?string
|
|
{
|
|
if (empty(trim($value))) {
|
|
return null;
|
|
}
|
|
|
|
try {
|
|
// Try to parse various date formats
|
|
$date = \Carbon\Carbon::parse($value);
|
|
return $date->format('Y-m-d');
|
|
} catch (\Exception $e) {
|
|
Log::warning('SlikImport: Invalid date format', [
|
|
'value' => $value,
|
|
'error' => $e->getMessage()
|
|
]);
|
|
return null;
|
|
}
|
|
}
|
|
}
|