'UTF-8', 'delimiter' => ',', 'enclosure' => '"', 'escape_character' => '\\', ]; } /** * Process collection data dari Excel dengan optimasi memory * * @param Collection $collection * @return void */ public function collection(Collection $collection) { // Set memory limit dari konfigurasi $memoryLimit = config('import.slik.memory_limit', 1024); $currentMemoryLimit = ini_get('memory_limit'); if ($currentMemoryLimit !== '-1' && $this->convertToBytes($currentMemoryLimit) < $memoryLimit * 1024 * 1024) { ini_set('memory_limit', $memoryLimit . 'M'); } // Set timeout handler $timeout = config('import.slik.timeout', 1800); set_time_limit($timeout); // Force garbage collection sebelum memulai if (config('import.slik.enable_gc', true)) { gc_collect_cycles(); } Log::info('SlikImport: Memulai import data', [ 'total_rows' => $collection->count(), 'memory_usage' => memory_get_usage(true), 'memory_peak' => memory_get_peak_usage(true), 'memory_limit' => ini_get('memory_limit'), 'php_version' => PHP_VERSION, 'memory_limit_before' => $currentMemoryLimit, 'config' => [ 'memory_limit' => $memoryLimit, 'chunk_size' => $this->chunkSize(), 'batch_size' => $this->batchSize() ] ]); DB::beginTransaction(); try { $processedRows = 0; $skippedRows = 0; $errorRows = 0; $totalRows = $collection->count(); foreach ($collection as $index => $row) { // Log progress setiap 25 baris untuk chunk lebih kecil if ($index % 25 === 0) { Log::info('SlikImport: Processing chunk', [ 'current_row' => $index + 5, 'progress' => round(($index / max($totalRows, 1)) * 100, 2) . '%', 'processed' => $processedRows, 'skipped' => $skippedRows, 'errors' => $errorRows, 'memory_usage' => memory_get_usage(true), 'memory_peak' => memory_get_peak_usage(true), 'memory_diff' => memory_get_peak_usage(true) - memory_get_usage(true) ]); } // Skip baris kosong if ($this->isEmptyRow($row)) { $skippedRows++; Log::debug('SlikImport: Skipping empty row', ['row_number' => $index + 5]); continue; } // Validasi data if (!$this->validateRow($row)) { $errorRows++; Log::warning('SlikImport: Invalid row data', [ 'row_number' => $index + 5, 'data' => $row->toArray() ]); continue; } try { // Map data dari Excel ke model $slikData = $this->mapRowToSlik($row); // Update atau create berdasarkan no_rekening dan cif $slik = Slik::updateOrCreate( [ 'no_rekening' => $slikData['no_rekening'], 'cif' => $slikData['cif'] ], $slikData ); $processedRows++; // Log detail untuk baris pertama sebagai sample if ($index === 0) { Log::info('SlikImport: Sample data processed', [ 'slik_id' => $slik->id, 'no_rekening' => $slik->no_rekening, 'cif' => $slik->cif, 'was_recently_created' => $slik->wasRecentlyCreated ]); } // Force garbage collection setiap 25 baris untuk mengurangi memory if (config('import.slik.enable_gc', true) && $index > 0 && $index % 25 === 0) { gc_collect_cycles(); } // Unset data yang sudah tidak digunakan untuk mengurangi memory if ($index > 0 && $index % 25 === 0) { unset($slikData, $slik); } // Reset collection internal untuk mengurangi memory if ($index > 0 && $index % 100 === 0) { $collection = collect($collection->slice($index + 1)->values()); } } catch (\Exception $e) { $errorRows++; Log::error('SlikImport: Error processing row', [ 'row_number' => $index + 5, 'error' => $e->getMessage(), 'data' => $row->toArray(), 'memory_usage' => memory_get_usage(true) ]); } } DB::commit(); // Force garbage collection setelah selesai if (config('import.slik.enable_gc', true)) { gc_collect_cycles(); } // Cleanup variables unset($collection); Log::info('SlikImport: Import berhasil diselesaikan', [ 'total_rows' => $totalRows, 'processed_rows' => $processedRows, 'skipped_rows' => $skippedRows, 'error_rows' => $errorRows, 'final_memory_usage' => memory_get_usage(true), 'peak_memory_usage' => memory_get_peak_usage(true), 'memory_saved' => memory_get_peak_usage(true) - memory_get_usage(true), 'memory_efficiency' => ($processedRows > 0) ? round(memory_get_peak_usage(true) / $processedRows, 2) : 0 ]); } catch (\Exception $e) { DB::rollBack(); // Force garbage collection jika error if (config('import.slik.enable_gc', true)) { gc_collect_cycles(); } $errorType = 'general'; if (str_contains(strtolower($e->getMessage()), 'memory')) { $errorType = 'memory'; } elseif (str_contains(strtolower($e->getMessage()), 'timeout') || str_contains(strtolower($e->getMessage()), 'maximum execution time')) { $errorType = 'timeout'; } Log::error('SlikImport: Error during import', [ 'error' => $e->getMessage(), 'error_type' => $errorType, 'exception_type' => get_class($e), 'trace' => $e->getTraceAsString(), 'file' => $e->getFile(), 'line' => $e->getLine(), 'memory_usage' => memory_get_usage(true), 'memory_peak' => memory_get_peak_usage(true), 'memory_limit' => ini_get('memory_limit'), 'timeout_limit' => ini_get('max_execution_time'), 'is_memory_error' => str_contains(strtolower($e->getMessage()), 'memory'), 'is_timeout_error' => str_contains(strtolower($e->getMessage()), 'timeout') || str_contains(strtolower($e->getMessage()), 'maximum execution time') ]); throw $e; } } /** * Convert memory limit string ke bytes * * @param string $memoryLimit * @return int */ private function convertToBytes(string $memoryLimit): int { $memoryLimit = trim($memoryLimit); $lastChar = strtolower(substr($memoryLimit, -1)); $number = (int) substr($memoryLimit, 0, -1); switch ($lastChar) { case 'g': return $number * 1024 * 1024 * 1024; case 'm': return $number * 1024 * 1024; case 'k': return $number * 1024; default: return (int) $memoryLimit; } } /** * Cek apakah baris kosong * * @param Collection $row * @return bool */ private function isEmptyRow(Collection $row): bool { return $row->filter(function ($value) { return !empty(trim($value)); })->isEmpty(); } /** * Validasi data baris * * @param Collection $row * @return bool */ private function validateRow(Collection $row): bool { // Validasi minimal: sandi_bank, no_rekening, dan cif harus ada return !empty(trim($row[0])) && // sandi_bank !empty(trim($row[5])) && // no_rekening !empty(trim($row[6])); // cif } /** * Map data dari baris Excel ke array untuk model Slik * * @param Collection $row * @return array */ private function mapRowToSlik(Collection $row): array { return [ 'sandi_bank' => trim($row[0]) ?: null, 'tahun' => $this->parseInteger($row[1]), 'bulan' => $this->parseInteger($row[2]), 'flag_detail' => trim($row[3]) ?: null, 'kode_register_agunan' => trim($row[4]) ?: null, 'no_rekening' => trim($row[5]) ?: null, 'cif' => trim($row[6]) ?: null, 'kolektibilitas' => trim($row[7]) ?: null, 'fasilitas' => trim($row[8]) ?: null, 'jenis_segmen_fasilitas' => trim($row[9]) ?: null, 'status_agunan' => trim($row[10]) ?: null, 'jenis_agunan' => trim($row[11]) ?: null, 'peringkat_agunan' => trim($row[12]) ?: null, 'lembaga_pemeringkat' => trim($row[13]) ?: null, 'jenis_pengikatan' => trim($row[14]) ?: null, 'tanggal_pengikatan' => $this->parseDate($row[15]), 'nama_pemilik_agunan' => trim($row[16]) ?: null, 'bukti_kepemilikan' => trim($row[17]) ?: null, 'alamat_agunan' => trim($row[18]) ?: null, 'lokasi_agunan' => trim($row[19]) ?: null, 'nilai_agunan' => $this->parseDecimal($row[20]), 'nilai_agunan_menurut_ljk' => $this->parseDecimal($row[21]), 'tanggal_penilaian_ljk' => $this->parseDate($row[22]), 'nilai_agunan_penilai_independen' => $this->parseDecimal($row[23]), 'nama_penilai_independen' => trim($row[24]) ?: null, 'tanggal_penilaian_penilai_independen' => $this->parseDate($row[25]), 'jumlah_hari_tunggakan' => $this->parseInteger($row[26]), 'status_paripasu' => trim($row[27]) ?: null, 'prosentase_paripasu' => $this->parseDecimal($row[28]), 'status_kredit_join' => trim($row[29]) ?: null, 'diasuransikan' => trim($row[30]) ?: null, 'keterangan' => trim($row[31]) ?: null, 'kantor_cabang' => trim($row[32]) ?: null, 'operasi_data' => trim($row[33]) ?: null, 'kode_cabang' => trim($row[34]) ?: null, 'nama_debitur' => trim($row[35]) ?: null, 'nama_cabang' => trim($row[36]) ?: null, 'flag' => trim($row[37]) ?: null, ]; } /** * Parse integer value * * @param mixed $value * @return int|null */ private function parseInteger($value): ?int { if (empty(trim($value))) { return null; } return (int) $value; } /** * Parse decimal value * * @param mixed $value * @return float|null */ private function parseDecimal($value): ?float { if (empty(trim($value))) { return null; } // Remove currency formatting $cleaned = str_replace([',', '.'], ['', '.'], $value); $cleaned = preg_replace('/[^0-9.]/', '', $cleaned); return (float) $cleaned; } /** * Parse date value * * @param mixed $value * @return string|null */ private function parseDate($value): ?string { if (empty(trim($value))) { return null; } try { // Try to parse various date formats $date = \Carbon\Carbon::parse($value); return $date->format('Y-m-d'); } catch (\Exception $e) { Log::warning('SlikImport: Invalid date format', [ 'value' => $value, 'error' => $e->getMessage() ]); return null; } } }