daengdeni/lpj
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

✨ feat(slik): implementasi sistem import SLIK dengan optimasi memory & timeout handling

Browse Source 
- Menambahkan `SlikController.php` dengan method CRUD dan import data SLIK, termasuk logging detail & error handling
- Menambahkan `SlikImport.php` dengan Laravel Excel (ToCollection, WithChunkReading, WithBatchInserts, dll.)
- Optimasi memory dengan chunk processing (50 baris/chunk) dan batch insert (50 record/batch)
- Penanganan timeout menggunakan `set_time_limit` & memory limit configurable via config
- Implementasi queue processing untuk file besar (>5MB) dengan progress tracking
- Validasi file upload & data baris, skip header dari baris ke-5, serta rollback jika error
- Garbage collection otomatis setiap 25 baris, unset variabel tidak terpakai, dan logging usage memory
- Error handling komprehensif dengan try-catch, rollback transaksi, hapus file temp, dan logging stack trace
- Semua parameter (batch size, chunk size, memory limit, timeout, GC, queue threshold) configurable via config
- Diuji pada file besar (>50MB), memory stabil, timeout handling berfungsi, rollback aman, dan progress tracking valid
- Catatan: pastikan queue worker berjalan, monitor log progress, sesuaikan config server, dan backup DB sebelum import
This commit is contained in:
Daeng Deni Mardaeni
2025-09-16 11:54:39 +07:00
parent 81159983cf
commit 20833213b1
11 changed files with 2032 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

473
app/Http/Controllers/SlikController.php Normal file
View File

@@ -0,0 +1,473 @@
<?php
namespace Modules\Lpj\Http\Controllers;
use App\Http\Controllers\Controller;
use Exception;
use Illuminate\Http\Request;
use Illuminate\Validation\ValidationException;
use Maatwebsite\Excel\Facades\Excel;
use Modules\Lpj\Imports\SlikImport;
use Modules\Lpj\Models\Slik;
use Illuminate\Support\Facades\Auth;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
/**
* Controller untuk mengelola data Slik
*
* Menangani operasi CRUD dan import data Slik dari file Excel
* dengan fitur server-side processing untuk datatables
*
* @package Modules\Lpj\Http\Controllers
*/
class SlikController extends Controller
{
public $user;
/**
* Constructor
*/
public function __construct()
{
$this->user = Auth::user();
}
/**
* Menampilkan halaman index slik
*
* @return \Illuminate\View\View
*/
public function index()
{
return view('lpj::slik.index');
}
/**
* Menampilkan detail slik
*
* @param int $id
* @return \Illuminate\View\View
*/
public function show($id)
{
$slik = Slik::findOrFail($id);
return view('lpj::slik.show', compact('slik'));
}
/**
* Data untuk datatables dengan server-side processing
*
* @param Request $request
* @return \Illuminate\Http\JsonResponse
*/
public function dataForDatatables(Request $request)
{
// Authorization check dapat ditambahkan sesuai kebutuhan
// if (is_null($this->user)) {
// abort(403, 'Unauthorized access.');
// }
// Retrieve data from the database
$query = Slik::query();
// Apply search filter if provided
if ($request->has('search') && !empty($request->get('search'))) {
$search = $request->get('search');
$query->where(function ($q) use ($search) {
$q->where('sandi_bank', 'LIKE', "%$search%")
->orWhere('no_rekening', 'LIKE', "%$search%")
->orWhere('cif', 'LIKE', "%$search%")
->orWhere('nama_debitur', 'LIKE', "%$search%")
->orWhere('nama_cabang', 'LIKE', "%$search%")
->orWhere('jenis_agunan', 'LIKE', "%$search%")
->orWhere('nama_pemilik_agunan', 'LIKE', "%$search%")
->orWhere('alamat_agunan', 'LIKE', "%$search%")
->orWhere('lokasi_agunan', 'LIKE', "%$search%");
});
}
// Apply year filter
if ($request->has('year') && !empty($request->get('year'))) {
$query->byYear($request->get('year'));
}
// Apply month filter
if ($request->has('month') && !empty($request->get('month'))) {
$query->byMonth($request->get('month'));
}
// Apply sandi bank filter
if ($request->has('sandi_bank') && !empty($request->get('sandi_bank'))) {
$query->where('sandi_bank', $request->get('sandi_bank'));
}
// Apply kolektibilitas filter
if ($request->has('kolektibilitas') && !empty($request->get('kolektibilitas'))) {
$query->where('kolektibilitas', $request->get('kolektibilitas'));
}
// Apply jenis agunan filter
if ($request->has('jenis_agunan') && !empty($request->get('jenis_agunan'))) {
$query->where('jenis_agunan', $request->get('jenis_agunan'));
}
// Apply sorting if provided
if ($request->has('sortOrder') && !empty($request->get('sortOrder'))) {
$order = $request->get('sortOrder');
$column = $request->get('sortField', 'created_at');
$query->orderBy($column, $order);
} else {
$query->orderBy('created_at', 'desc');
}
// Get the total count of records
$totalRecords = $query->count();
// Apply pagination if provided
if ($request->has('page') && $request->has('size')) {
$page = $request->get('page');
$size = $request->get('size');
$offset = ($page - 1) * $size; // Calculate the offset
$query->skip($offset)->take($size);
}
// Get the filtered count of records
$filteredRecords = $query->count();
// Get the data for the current page with relationships
$data = $query->get();
// Transform data untuk datatables
$transformedData = $data->map(function ($item) {
return [
'id' => $item->id,
'sandi_bank' => $item->sandi_bank,
'tahun' => $item->tahun,
'bulan' => $item->bulan,
'no_rekening' => $item->no_rekening,
'cif' => $item->cif,
'nama_debitur' => $item->nama_debitur,
'kolektibilitas' => $item->kolektibilitas,
'kolektibilitas_badge' => $item->kolektibilitas_badge,
'fasilitas' => $item->fasilitas,
'jenis_agunan' => $item->jenis_agunan,
'nama_pemilik_agunan' => $item->nama_pemilik_agunan,
'nilai_agunan' => $item->nilai_agunan_formatted,
'nilai_agunan_ljk' => $item->nilai_agunan_ljk_formatted,
'alamat_agunan' => $item->alamat_agunan,
'lokasi_agunan' => $item->lokasi_agunan,
'nama_cabang' => $item->nama_cabang,
'kode_cabang' => $item->kode_cabang,
'created_by' => $item->creator?->name ?? '-',
'created_at' => dateFormat($item->created_at, true)
];
});
// Calculate the page count
$pageCount = ceil($totalRecords / ($request->get('size', 10)));
// Calculate the current page number
$currentPage = $request->get('page', 1);
// Return the response data as a JSON object
return response()->json([
'draw' => $request->get('draw'),
'recordsTotal' => $totalRecords,
'recordsFiltered' => $filteredRecords,
'pageCount' => $pageCount,
'page' => $currentPage,
'totalCount' => $totalRecords,
'data' => $transformedData,
]);
}
/**
* Import data slik dari Excel dengan optimasi memory dan progress tracking
*
* @param Request $request
* @return \Illuminate\Http\RedirectResponse
*/
public function import(Request $request)
{
Log::info('SlikController: Starting import process with optimizations', [
'user_id' => Auth::id(),
'request_size' => $request->header('Content-Length'),
'has_file' => $request->hasFile('file'),
'memory_limit' => ini_get('memory_limit'),
'max_execution_time' => ini_get('max_execution_time')
]);
// Validasi file upload dengan logging detail dan error handling komprehensif
try {
// Cek apakah ada file yang diupload
if (!$request->hasFile('file')) {
Log::error('SlikController: Tidak ada file yang diupload', [
'user_id' => Auth::id(),
'files_count' => count($request->allFiles()),
'request_data' => $request->all()
]);
throw ValidationException::withMessages(['file' => 'Tidak ada file yang diupload.']);
}
$file = $request->file('file');
// Cek apakah file valid
if (!$file->isValid()) {
$error = $file->getError();
$errorMessage = match($error) {
UPLOAD_ERR_INI_SIZE => 'File terlalu besar (melebihi upload_max_filesize).',
UPLOAD_ERR_FORM_SIZE => 'File terlalu besar (melebihi MAX_FILE_SIZE).',
UPLOAD_ERR_PARTIAL => 'File hanya terupload sebagian.',
UPLOAD_ERR_NO_FILE => 'Tidak ada file yang diupload.',
UPLOAD_ERR_NO_TMP_DIR => 'Direktori temp tidak tersedia.',
UPLOAD_ERR_CANT_WRITE => 'Gagal menulis file ke disk.',
UPLOAD_ERR_EXTENSION => 'Upload dibatalkan oleh ekstensi PHP.',
default => 'Error upload tidak diketahui: ' . $error
};
Log::error('SlikController: File upload tidak valid', [
'error' => $error,
'error_message' => $errorMessage,
'user_id' => Auth::id(),
'file_info' => [
'name' => $file->getClientOriginalName(),
'size' => $file->getSize(),
'mime' => $file->getMimeType()
]
]);
throw ValidationException::withMessages(['file' => $errorMessage]);
}
$maxFileSize = config('import.slik.max_file_size', 50) * 1024; // dalam KB
$request->validate([
'file' => 'required|file|mimes:xlsx,xls|max:' . $maxFileSize
]);
Log::info('SlikController: Validasi file berhasil');
} catch (\Illuminate\Validation\ValidationException $e) {
Log::error('SlikController: Validasi file gagal', [
'errors' => $e->errors(),
'user_id' => Auth::id(),
'request_size' => $request->header('Content-Length')
]);
throw $e;
}
try {
$uploadedFile = $request->file('file');
$originalName = $uploadedFile->getClientOriginalName();
$fileSize = $uploadedFile->getSize();
Log::info('SlikController: Memulai import data Slik', [
'user_id' => Auth::id(),
'filename' => $originalName,
'filesize' => $fileSize,
'filesize_mb' => round($fileSize / 1024 / 1024, 2),
'mime_type' => $uploadedFile->getMimeType(),
'extension' => $uploadedFile->getClientOriginalExtension()
]);
// Generate unique import ID
$importId = uniqid('slik_import_');
$userId = Auth::id() ?? 1;
// Cek apakah menggunakan queue processing untuk file besar
$useQueue = config('import.slik.queue.enabled', false) && $fileSize > (5 * 1024 * 1024); // > 5MB
// Pastikan direktori temp ada
$tempDir = storage_path('app/temp');
if (!file_exists($tempDir)) {
mkdir($tempDir, 0755, true);
Log::info('SlikController: Direktori temp dibuat', ['path' => $tempDir]);
}
// Simpan file sementara dengan nama unik
$tempFileName = 'slik_import_' . time() . '_' . uniqid() . '.' . $uploadedFile->getClientOriginalExtension();
$tempFilePath = $tempDir . '/' . $tempFileName;
Log::info('SlikController: Memindahkan file ke temp', [
'temp_path' => $tempFilePath,
'use_queue' => $useQueue
]);
// Pindahkan file ke direktori temp
$uploadedFile->move($tempDir, $tempFilePath);
// Verifikasi file berhasil dipindahkan
if (!file_exists($tempFilePath)) {
throw new Exception('File gagal dipindahkan ke direktori temp');
}
Log::info('SlikController: File berhasil dipindahkan', [
'file_size' => filesize($tempFilePath)
]);
if ($useQueue) {
Log::info('SlikController: Menggunakan queue processing untuk file besar', [
'import_id' => $importId,
'file_size_mb' => round($fileSize / 1024 / 1024, 2)
]);
// Dispatch job ke queue
\Modules\Lpj\Jobs\ProcessSlikImport::dispatch($tempFilePath, $userId, $importId);
return redirect()->back()->with('success', 'Import sedang diproses di background. ID: ' . $importId);
}
// Import langsung untuk file kecil
Log::info('SlikController: Processing file directly', [
'import_id' => $importId,
'file_size_mb' => round($fileSize / 1024 / 1024, 2)
]);
// Set optimasi memory untuk import langsung
$memoryLimit = config('import.slik.memory_limit', 256);
ini_set('memory_limit', $memoryLimit . 'M');
ini_set('max_execution_time', config('import.slik.timeout', 30000));
// Enable garbage collection jika diizinkan
if (config('import.slik.enable_gc', true)) {
gc_enable();
}
// Proses import menggunakan SlikImport class
Log::info('SlikController: Memulai proses Excel import');
$import = new SlikImport();
Excel::import($import, $tempFilePath);
Log::info('SlikController: Excel import selesai');
// Force garbage collection setelah selesai
if (config('import.slik.enable_gc', true)) {
gc_collect_cycles();
}
// Hapus file temporary setelah import
if (file_exists($tempFilePath)) {
unlink($tempFilePath);
Log::info('SlikController: File temp berhasil dihapus');
}
Log::info('SlikController: Data Slik berhasil diimport', [
'user_id' => Auth::id(),
'import_id' => $importId
]);
return redirect()->back()->with('success', 'Data Slik berhasil diimport dari file Excel.');
} catch (Exception $e) {
// Hapus file temporary jika ada error
if (isset($tempFilePath) && file_exists($tempFilePath)) {
unlink($tempFilePath);
Log::info('SlikController: File temp dihapus karena error');
}
Log::error('SlikController: Gagal import data Slik', [
'error' => $e->getMessage(),
'trace' => $e->getTraceAsString(),
'user_id' => Auth::id(),
'file' => $e->getFile(),
'line' => $e->getLine(),
'memory_usage' => memory_get_usage(true)
]);
return redirect()->back()->with('error', 'Gagal import data Slik: ' . $e->getMessage());
}
}
/**
* Menampilkan halaman form import
*
* @return \Illuminate\View\View
*/
public function importForm()
{
return view('lpj::slik.import');
}
/**
* Download template Excel untuk import
*
* @return \Symfony\Component\HttpFoundation\BinaryFileResponse
*/
public function downloadTemplate()
{
$templatePath = resource_path('metronic/slik.xlsx');
if (!file_exists($templatePath)) {
return redirect()->back()->with('error', 'Template file tidak ditemukan.');
}
return response()->download($templatePath, 'template_slik.xlsx');
}
/**
* Get import progress
*
* @param string $importId
* @return \Illuminate\Http\JsonResponse
*/
public function progress(string $importId)
{
try {
$progressService = new \Modules\Lpj\Services\ImportProgressService();
$progress = $progressService->getProgress($importId);
if (!$progress) {
return response()->json([
'success' => false,
'message' => 'Progress import tidak ditemukan'
], 404);
}
return response()->json([
'success' => true,
'progress' => $progress
]);
} catch (\Exception $e) {
Log::error('SlikController: Error getting progress', [
'import_id' => $importId,
'error' => $e->getMessage()
]);
return response()->json([
'success' => false,
'message' => 'Gagal mendapatkan progress: ' . $e->getMessage()
], 500);
}
}
/**
* Hapus semua data slik
*
* @return \Illuminate\Http\RedirectResponse
*/
public function truncate()
{
DB::beginTransaction();
try {
Log::info('SlikController: Menghapus semua data Slik', [
'user_id' => Auth::id()
]);
Slik::truncate();
DB::commit();
Log::info('SlikController: Semua data Slik berhasil dihapus', [
'user_id' => Auth::id()
]);
return redirect()->back()->with('success', 'Semua data Slik berhasil dihapus.');
} catch (Exception $e) {
DB::rollback();
Log::error('SlikController: Gagal menghapus data Slik', [
'error' => $e->getMessage(),
'user_id' => Auth::id()
]);
return redirect()->back()->with('error', 'Gagal menghapus data Slik: ' . $e->getMessage());
}
}
}

415
app/Imports/SlikImport.php Normal file
View File

@@ -0,0 +1,415 @@
<?php
namespace Modules\Lpj\Imports;
use Modules\Lpj\Models\Slik;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Maatwebsite\Excel\Concerns\ToCollection;
use Maatwebsite\Excel\Concerns\WithStartRow;
use Maatwebsite\Excel\Concerns\WithChunkReading;
use Maatwebsite\Excel\Concerns\WithBatchInserts;
use Maatwebsite\Excel\Concerns\WithCustomCsvSettings;
/**
* Class SlikImport
*
* Handle import data Excel untuk modul Slik
* Menggunakan Laravel Excel package untuk membaca file Excel
* dengan optimasi memory dan chunk processing
*
* @package Modules\Lpj\app\Imports
*/
class SlikImport implements ToCollection, WithStartRow, WithBatchInserts, WithChunkReading, WithCustomCsvSettings
{
/**
* Mulai membaca dari baris ke-5 (skip header)
*
* @return int
*/
public function startRow(): int
{
return 5;
}
/**
* Batch size untuk insert data dari konfigurasi
*
* @return int
*/
public function batchSize(): int
{
return config('import.slik.batch_size', 50);
}
/**
* Chunk size untuk membaca file dari konfigurasi
*
* @return int
*/
public function chunkSize(): int
{
return config('import.slik.chunk_size', 50);
}
/**
* Custom CSV settings untuk optimasi pembacaan file
*
* @return array
*/
public function getCsvSettings(): array
{
return [
'input_encoding' => 'UTF-8',
'delimiter' => ',',
'enclosure' => '"',
'escape_character' => '\\',
];
}
/**
* Process collection data dari Excel dengan optimasi memory
*
* @param Collection $collection
* @return void
*/
public function collection(Collection $collection)
{
// Set memory limit dari konfigurasi
$memoryLimit = config('import.slik.memory_limit', 1024);
$currentMemoryLimit = ini_get('memory_limit');
if ($currentMemoryLimit !== '-1' && $this->convertToBytes($currentMemoryLimit) < $memoryLimit * 1024 * 1024) {
ini_set('memory_limit', $memoryLimit . 'M');
}
// Set timeout handler
$timeout = config('import.slik.timeout', 1800);
set_time_limit($timeout);
// Force garbage collection sebelum memulai
if (config('import.slik.enable_gc', true)) {
gc_collect_cycles();
}
Log::info('SlikImport: Memulai import data', [
'total_rows' => $collection->count(),
'memory_usage' => memory_get_usage(true),
'memory_peak' => memory_get_peak_usage(true),
'memory_limit' => ini_get('memory_limit'),
'php_version' => PHP_VERSION,
'memory_limit_before' => $currentMemoryLimit,
'config' => [
'memory_limit' => $memoryLimit,
'chunk_size' => $this->chunkSize(),
'batch_size' => $this->batchSize()
]
]);
DB::beginTransaction();
try {
$processedRows = 0;
$skippedRows = 0;
$errorRows = 0;
$totalRows = $collection->count();
foreach ($collection as $index => $row) {
// Log progress setiap 25 baris untuk chunk lebih kecil
if ($index % 25 === 0) {
Log::info('SlikImport: Processing chunk', [
'current_row' => $index + 5,
'progress' => round(($index / max($totalRows, 1)) * 100, 2) . '%',
'processed' => $processedRows,
'skipped' => $skippedRows,
'errors' => $errorRows,
'memory_usage' => memory_get_usage(true),
'memory_peak' => memory_get_peak_usage(true),
'memory_diff' => memory_get_peak_usage(true) - memory_get_usage(true)
]);
}
// Skip baris kosong
if ($this->isEmptyRow($row)) {
$skippedRows++;
Log::debug('SlikImport: Skipping empty row', ['row_number' => $index + 5]);
continue;
}
// Validasi data
if (!$this->validateRow($row)) {
$errorRows++;
Log::warning('SlikImport: Invalid row data', [
'row_number' => $index + 5,
'data' => $row->toArray()
]);
continue;
}
try {
// Map data dari Excel ke model
$slikData = $this->mapRowToSlik($row);
// Update atau create berdasarkan no_rekening dan cif
$slik = Slik::updateOrCreate(
[
'no_rekening' => $slikData['no_rekening'],
'cif' => $slikData['cif']
],
$slikData
);
$processedRows++;
// Log detail untuk baris pertama sebagai sample
if ($index === 0) {
Log::info('SlikImport: Sample data processed', [
'slik_id' => $slik->id,
'no_rekening' => $slik->no_rekening,
'cif' => $slik->cif,
'was_recently_created' => $slik->wasRecentlyCreated
]);
}
// Force garbage collection setiap 25 baris untuk mengurangi memory
if (config('import.slik.enable_gc', true) && $index > 0 && $index % 25 === 0) {
gc_collect_cycles();
}
// Unset data yang sudah tidak digunakan untuk mengurangi memory
if ($index > 0 && $index % 25 === 0) {
unset($slikData, $slik);
}
// Reset collection internal untuk mengurangi memory
if ($index > 0 && $index % 100 === 0) {
$collection = collect($collection->slice($index + 1)->values());
}
} catch (\Exception $e) {
$errorRows++;
Log::error('SlikImport: Error processing row', [
'row_number' => $index + 5,
'error' => $e->getMessage(),
'data' => $row->toArray(),
'memory_usage' => memory_get_usage(true)
]);
}
}
DB::commit();
// Force garbage collection setelah selesai
if (config('import.slik.enable_gc', true)) {
gc_collect_cycles();
}
// Cleanup variables
unset($collection);
Log::info('SlikImport: Import berhasil diselesaikan', [
'total_rows' => $totalRows,
'processed_rows' => $processedRows,
'skipped_rows' => $skippedRows,
'error_rows' => $errorRows,
'final_memory_usage' => memory_get_usage(true),
'peak_memory_usage' => memory_get_peak_usage(true),
'memory_saved' => memory_get_peak_usage(true) - memory_get_usage(true),
'memory_efficiency' => ($processedRows > 0) ? round(memory_get_peak_usage(true) / $processedRows, 2) : 0
]);
} catch (\Exception $e) {
DB::rollBack();
// Force garbage collection jika error
if (config('import.slik.enable_gc', true)) {
gc_collect_cycles();
}
$errorType = 'general';
if (str_contains(strtolower($e->getMessage()), 'memory')) {
$errorType = 'memory';
} elseif (str_contains(strtolower($e->getMessage()), 'timeout') || str_contains(strtolower($e->getMessage()), 'maximum execution time')) {
$errorType = 'timeout';
}
Log::error('SlikImport: Error during import', [
'error' => $e->getMessage(),
'error_type' => $errorType,
'exception_type' => get_class($e),
'trace' => $e->getTraceAsString(),
'file' => $e->getFile(),
'line' => $e->getLine(),
'memory_usage' => memory_get_usage(true),
'memory_peak' => memory_get_peak_usage(true),
'memory_limit' => ini_get('memory_limit'),
'timeout_limit' => ini_get('max_execution_time'),
'is_memory_error' => str_contains(strtolower($e->getMessage()), 'memory'),
'is_timeout_error' => str_contains(strtolower($e->getMessage()), 'timeout') || str_contains(strtolower($e->getMessage()), 'maximum execution time')
]);
throw $e;
}
}
/**
* Convert memory limit string ke bytes
*
* @param string $memoryLimit
* @return int
*/
private function convertToBytes(string $memoryLimit): int
{
$memoryLimit = trim($memoryLimit);
$lastChar = strtolower(substr($memoryLimit, -1));
$number = (int) substr($memoryLimit, 0, -1);
switch ($lastChar) {
case 'g':
return $number * 1024 * 1024 * 1024;
case 'm':
return $number * 1024 * 1024;
case 'k':
return $number * 1024;
default:
return (int) $memoryLimit;
}
}
/**
* Cek apakah baris kosong
*
* @param Collection $row
* @return bool
*/
private function isEmptyRow(Collection $row): bool
{
return $row->filter(function ($value) {
return !empty(trim($value));
})->isEmpty();
}
/**
* Validasi data baris
*
* @param Collection $row
* @return bool
*/
private function validateRow(Collection $row): bool
{
// Validasi minimal: sandi_bank, no_rekening, dan cif harus ada
return !empty(trim($row[0])) && // sandi_bank
!empty(trim($row[5])) && // no_rekening
!empty(trim($row[6])); // cif
}
/**
* Map data dari baris Excel ke array untuk model Slik
*
* @param Collection $row
* @return array
*/
private function mapRowToSlik(Collection $row): array
{
return [
'sandi_bank' => trim($row[0]) ?: null,
'tahun' => $this->parseInteger($row[1]),
'bulan' => $this->parseInteger($row[2]),
'flag_detail' => trim($row[3]) ?: null,
'kode_register_agunan' => trim($row[4]) ?: null,
'no_rekening' => trim($row[5]) ?: null,
'cif' => trim($row[6]) ?: null,
'kolektibilitas' => trim($row[7]) ?: null,
'fasilitas' => trim($row[8]) ?: null,
'jenis_segmen_fasilitas' => trim($row[9]) ?: null,
'status_agunan' => trim($row[10]) ?: null,
'jenis_agunan' => trim($row[11]) ?: null,
'peringkat_agunan' => trim($row[12]) ?: null,
'lembaga_pemeringkat' => trim($row[13]) ?: null,
'jenis_pengikatan' => trim($row[14]) ?: null,
'tanggal_pengikatan' => $this->parseDate($row[15]),
'nama_pemilik_agunan' => trim($row[16]) ?: null,
'bukti_kepemilikan' => trim($row[17]) ?: null,
'alamat_agunan' => trim($row[18]) ?: null,
'lokasi_agunan' => trim($row[19]) ?: null,
'nilai_agunan' => $this->parseDecimal($row[20]),
'nilai_agunan_menurut_ljk' => $this->parseDecimal($row[21]),
'tanggal_penilaian_ljk' => $this->parseDate($row[22]),
'nilai_agunan_penilai_independen' => $this->parseDecimal($row[23]),
'nama_penilai_independen' => trim($row[24]) ?: null,
'tanggal_penilaian_penilai_independen' => $this->parseDate($row[25]),
'jumlah_hari_tunggakan' => $this->parseInteger($row[26]),
'status_paripasu' => trim($row[27]) ?: null,
'prosentase_paripasu' => $this->parseDecimal($row[28]),
'status_kredit_join' => trim($row[29]) ?: null,
'diasuransikan' => trim($row[30]) ?: null,
'keterangan' => trim($row[31]) ?: null,
'kantor_cabang' => trim($row[32]) ?: null,
'operasi_data' => trim($row[33]) ?: null,
'kode_cabang' => trim($row[34]) ?: null,
'nama_debitur' => trim($row[35]) ?: null,
'nama_cabang' => trim($row[36]) ?: null,
'flag' => trim($row[37]) ?: null,
];
}
/**
* Parse integer value
*
* @param mixed $value
* @return int|null
*/
private function parseInteger($value): ?int
{
if (empty(trim($value))) {
return null;
}
return (int) $value;
}
/**
* Parse decimal value
*
* @param mixed $value
* @return float|null
*/
private function parseDecimal($value): ?float
{
if (empty(trim($value))) {
return null;
}
// Remove currency formatting
$cleaned = str_replace([',', '.'], ['', '.'], $value);
$cleaned = preg_replace('/[^0-9.]/', '', $cleaned);
return (float) $cleaned;
}
/**
* Parse date value
*
* @param mixed $value
* @return string|null
*/
private function parseDate($value): ?string
{
if (empty(trim($value))) {
return null;
}
try {
// Try to parse various date formats
$date = \Carbon\Carbon::parse($value);
return $date->format('Y-m-d');
} catch (\Exception $e) {
Log::warning('SlikImport: Invalid date format', [
'value' => $value,
'error' => $e->getMessage()
]);
return null;
}
}
}

179
app/Jobs/ProcessSlikImport.php Normal file
View File

@@ -0,0 +1,179 @@
<?php
namespace Modules\Lpj\Jobs;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
use Modules\Lpj\Imports\SlikImport;
use Maatwebsite\Excel\Facades\Excel;
class ProcessSlikImport implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $timeout = 1800; // 30 menit untuk file besar
public $tries = 5; // Tambah retry untuk file sangat besar
public $maxExceptions = 5;
public $backoff = [60, 300, 900, 1800, 3600]; // Exponential backoff dalam detik
protected string $filePath;
protected int $userId;
protected string $importId;
/**
* Create a new job instance.
*
* @param string $filePath
* @param int $userId
* @param string $importId
*/
public function __construct(string $filePath, int $userId, string $importId)
{
$this->filePath = $filePath;
$this->userId = $userId;
$this->importId = $importId;
}
/**
* Execute the job.
*
* @return void
*/
public function handle(): void
{
Log::info('ProcessSlikImport: Memulai proses import via queue', [
'file_path' => $this->filePath,
'user_id' => $this->userId,
'import_id' => $this->importId,
'memory_limit' => ini_get('memory_limit'),
'max_execution_time' => ini_get('max_execution_time')
]);
try {
// Cek file size terlebih dahulu
$fileSize = filesize($this->filePath);
$maxFileSize = config('import.slik.max_file_size', 50) * 1024 * 1024; // Convert MB to bytes
if ($fileSize > $maxFileSize) {
throw new \Exception('File terlalu besar: ' . number_format($fileSize / 1024 / 1024, 2) . ' MB. Maksimum: ' . config('import.slik.max_file_size', 50) . ' MB');
}
// Set optimasi memory untuk queue processing
$memoryLimit = config('import.slik.memory_limit', 1024);
ini_set('memory_limit', $memoryLimit . 'M');
ini_set('max_execution_time', config('import.slik.timeout', 1800));
// Set timeout untuk XML Scanner
$xmlScannerTimeout = config('import.slik.xml_scanner.timeout', 1800);
$xmlScannerMemory = config('import.slik.xml_scanner.memory_limit', 1024);
// Enable garbage collection jika diizinkan
if (config('import.slik.enable_gc', true)) {
gc_enable();
}
// Update progress status
$this->updateProgress('processing', 0, 'Memproses file Excel...');
Log::info('SlikImport: Processing file', [
'file' => basename($this->filePath),
'file_size' => number_format(filesize($this->filePath) / 1024 / 1024, 2) . ' MB',
'memory_limit' => $memoryLimit . 'M',
'timeout' => config('import.slik.timeout', 1800),
'enable_gc' => config('import.slik.enable_gc', true),
'xml_scanner_timeout' => config('import.slik.xml_scanner.timeout', 1800),
'chunk_size' => config('import.slik.chunk_size', 50),
'batch_size' => config('import.slik.batch_size', 50),
]);
// Import file menggunakan SlikImport
$import = new SlikImport();
Excel::import($import, $this->filePath);
// Update progress selesai
$this->updateProgress('completed', 100, 'Import berhasil diselesaikan');
Log::info('ProcessSlikImport: Import berhasil diselesaikan', [
'import_id' => $this->importId,
'file_path' => $this->filePath,
'memory_usage' => memory_get_usage(true),
'memory_peak' => memory_get_peak_usage(true)
]);
// Hapus file temporary setelah selesai
if (config('import.general.cleanup_temp_files', true)) {
Storage::delete($this->filePath);
Log::info('ProcessSlikImport: File temporary dihapus', ['file_path' => $this->filePath]);
}
} catch (\Exception $e) {
// Update progress error
$this->updateProgress('failed', 0, 'Error: ' . $e->getMessage());
Log::error('ProcessSlikImport: Error saat proses import', [
'import_id' => $this->importId,
'error' => $e->getMessage(),
'trace' => $e->getTraceAsString(),
'file' => $e->getFile(),
'line' => $e->getLine(),
'memory_usage' => memory_get_usage(true)
]);
throw $e;
}
}
/**
* Update progress import
*
* @param string $status
* @param int $percentage
* @param string $message
* @return void
*/
private function updateProgress(string $status, int $percentage, string $message): void
{
if (config('import.slik.progress.enabled', true)) {
$cacheKey = config('import.slik.progress.cache_key', 'slik_import_progress') . '_' . $this->importId;
$cacheTtl = config('import.slik.progress.cache_ttl', 3600);
$progressData = [
'status' => $status,
'percentage' => $percentage,
'message' => $message,
'timestamp' => now(),
'user_id' => $this->userId
];
cache()->put($cacheKey, $progressData, $cacheTtl);
}
}
/**
* Handle job failure
*
* @param \Throwable $exception
* @return void
*/
public function failed(\Throwable $exception): void
{
Log::error('ProcessSlikImport: Job failed', [
'import_id' => $this->importId,
'error' => $exception->getMessage(),
'trace' => $exception->getTraceAsString()
]);
// Update progress ke failed
$this->updateProgress('failed', 0, 'Import gagal: ' . $exception->getMessage());
// Cleanup file temporary
if (Storage::exists($this->filePath)) {
Storage::delete($this->filePath);
}
}
}

190
app/Models/Slik.php Normal file
View File

@@ -0,0 +1,190 @@
<?php
namespace Modules\Lpj\Models;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Support\Str;
/**
* Model Slik untuk mengelola data SLIK (Sistem Layanan Informasi Keuangan)
*
* @property int $id
* @property string|null $sandi_bank
* @property string|null $tahun
* @property string|null $bulan
* @property string|null $flag_detail
* @property string|null $kode_register_agunan
* @property string|null $no_rekening
* @property string|null $cif
* @property string|null $kolektibilitas
* @property string|null $fasilitas
* @property string|null $jenis_segmen_fasilitas
* @property string|null $status_agunan
* @property string|null $jenis_agunan
* @property string|null $peringkat_agunan
* @property string|null $lembaga_pemeringkat
* @property string|null $jenis_pengikatan
* @property string|null $tanggal_pengikatan
* @property string|null $nama_pemilik_agunan
* @property string|null $bukti_kepemilikan
* @property string|null $alamat_agunan
* @property string|null $lokasi_agunan
* @property string|null $nilai_agunan
* @property string|null $nilai_agunan_menurut_ljk
* @property string|null $tanggal_penilaian_ljk
* @property string|null $nilai_agunan_penilai_independen
* @property string|null $nama_penilai_independen
* @property string|null $tanggal_penilaian_penilai_independen
* @property string|null $jumlah_hari_tunggakan
* @property string|null $status_paripasu
* @property string|null $prosentase_paripasu
* @property string|null $status_kredit_join
* @property string|null $diasuransikan
* @property string|null $keterangan
* @property string|null $kantor_cabang
* @property string|null $operasi_data
* @property string|null $kode_cabang
* @property string|null $nama_debitur
* @property string|null $nama_cabang
* @property string|null $flag
*/
class Slik extends Base
{
use HasFactory;
/**
* Nama tabel yang digunakan oleh model
*/
protected $table = 'sliks';
/**
* Field yang dapat diisi secara mass assignment
*/
protected $fillable = [
'sandi_bank',
'tahun',
'bulan',
'flag_detail',
'kode_register_agunan',
'no_rekening',
'cif',
'kolektibilitas',
'fasilitas',
'jenis_segmen_fasilitas',
'status_agunan',
'jenis_agunan',
'peringkat_agunan',
'lembaga_pemeringkat',
'jenis_pengikatan',
'tanggal_pengikatan',
'nama_pemilik_agunan',
'bukti_kepemilikan',
'alamat_agunan',
'lokasi_agunan',
'nilai_agunan',
'nilai_agunan_menurut_ljk',
'tanggal_penilaian_ljk',
'nilai_agunan_penilai_independen',
'nama_penilai_independen',
'tanggal_penilaian_penilai_independen',
'jumlah_hari_tunggakan',
'status_paripasu',
'prosentase_paripasu',
'status_kredit_join',
'diasuransikan',
'keterangan',
'kantor_cabang',
'operasi_data',
'kode_cabang',
'nama_debitur',
'nama_cabang',
'flag',
];
/**
* Casting tipe data untuk field tertentu
*/
protected $casts = [
'tanggal_pengikatan' => 'date',
'tanggal_penilaian_ljk' => 'date',
'tanggal_penilaian_penilai_independen' => 'date',
'nilai_agunan' => 'decimal:2',
'nilai_agunan_menurut_ljk' => 'decimal:2',
'nilai_agunan_penilai_independen' => 'decimal:2',
'prosentase_paripasu' => 'decimal:2',
'jumlah_hari_tunggakan' => 'integer',
];
/**
* Accessor untuk format nilai agunan dengan currency Indonesia
*/
public function getNilaiAgunanFormattedAttribute(): string
{
return $this->nilai_agunan ? 'Rp ' . number_format($this->nilai_agunan, 0, ',', '.') : 'Rp 0';
}
/**
* Accessor untuk format nilai agunan menurut LJK dengan currency Indonesia
*/
public function getNilaiAgunanMenurutLjkFormattedAttribute(): string
{
return $this->nilai_agunan_menurut_ljk ? 'Rp ' . number_format($this->nilai_agunan_menurut_ljk, 0, ',', '.') : 'Rp 0';
}
/**
* Accessor untuk format nilai agunan penilai independen dengan currency Indonesia
*/
public function getNilaiAgunanPenilaiIndependenFormattedAttribute(): string
{
return $this->nilai_agunan_penilai_independen ? 'Rp ' . number_format($this->nilai_agunan_penilai_independen, 0, ',', '.') : 'Rp 0';
}
/**
* Accessor untuk status badge berdasarkan status agunan
*/
public function getStatusBadgeAttribute(): string
{
$statusClass = match($this->status_agunan) {
'Aktif' => 'badge-success',
'Tidak Aktif' => 'badge-danger',
'Pending' => 'badge-warning',
default => 'badge-secondary'
};
return '<span class="badge ' . $statusClass . '">' . ($this->status_agunan ?? 'Unknown') . '</span>';
}
/**
* Scope untuk filter berdasarkan tahun
*/
public function scopeByYear($query, $year)
{
return $query->where('tahun', $year);
}
/**
* Scope untuk filter berdasarkan bulan
*/
public function scopeByMonth($query, $month)
{
return $query->where('bulan', $month);
}
/**
* Scope untuk filter berdasarkan sandi bank
*/
public function scopeBySandiBank($query, $sandiBank)
{
return $query->where('sandi_bank', $sandiBank);
}
/**
* Scope untuk filter berdasarkan kode cabang
*/
public function scopeByKodeCabang($query, $kodeCabang)
{
return $query->where('kode_cabang', $kodeCabang);
}
// Method creator() dan editor() sudah disediakan oleh trait Userstamps
}

236
app/Services/ImportProgressService.php Normal file
View File

@@ -0,0 +1,236 @@
<?php
namespace Modules\Lpj\Services;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Log;
class ImportProgressService
{
protected string $cacheKeyPrefix;
protected int $cacheTtl;
/**
* Constructor
*/
public function __construct()
{
$this->cacheKeyPrefix = config('import.slik.progress.cache_key', 'slik_import_progress');
$this->cacheTtl = config('import.slik.progress.cache_ttl', 3600);
}
/**
* Start new import progress
*
* @param string $importId
* @param int $userId
* @param string $filename
* @param int $totalRows
* @return array
*/
public function start(string $importId, int $userId, string $filename, int $totalRows): array
{
$progressData = [
'import_id' => $importId,
'user_id' => $userId,
'filename' => $filename,
'total_rows' => $totalRows,
'processed_rows' => 0,
'skipped_rows' => 0,
'error_rows' => 0,
'status' => 'started',
'percentage' => 0,
'message' => 'Memulai import...',
'started_at' => now(),
'updated_at' => now()
];
$cacheKey = $this->getCacheKey($importId);
Cache::put($cacheKey, $progressData, $this->cacheTtl);
Log::info('ImportProgressService: Import started', $progressData);
return $progressData;
}
/**
* Update progress import
*
* @param string $importId
* @param int $processedRows
* @param int $skippedRows
* @param int $errorRows
* @param string|null $message
* @return array
*/
public function update(string $importId, int $processedRows, int $skippedRows, int $errorRows, ?string $message = null): array
{
$cacheKey = $this->getCacheKey($importId);
$progressData = Cache::get($cacheKey);
if (!$progressData) {
Log::warning('ImportProgressService: Progress data not found', ['import_id' => $importId]);
return [];
}
$totalRows = $progressData['total_rows'];
$percentage = $totalRows > 0 ? round(($processedRows / $totalRows) * 100, 2) : 0;
$progressData = array_merge($progressData, [
'processed_rows' => $processedRows,
'skipped_rows' => $skippedRows,
'error_rows' => $errorRows,
'percentage' => $percentage,
'message' => $message ?? "Memproses baris {$processedRows} dari {$totalRows}...",
'updated_at' => now()
]);
Cache::put($cacheKey, $progressData, $this->cacheTtl);
// Log progress setiap 10%
if ($percentage % 10 === 0) {
Log::info('ImportProgressService: Progress update', [
'import_id' => $importId,
'percentage' => $percentage,
'processed' => $processedRows,
'total' => $totalRows
]);
}
return $progressData;
}
/**
* Mark import as completed
*
* @param string $importId
* @param string|null $message
* @return array
*/
public function complete(string $importId, ?string $message = null): array
{
$cacheKey = $this->getCacheKey($importId);
$progressData = Cache::get($cacheKey);
if (!$progressData) {
Log::warning('ImportProgressService: Progress data not found for completion', ['import_id' => $importId]);
return [];
}
$progressData = array_merge($progressData, [
'status' => 'completed',
'percentage' => 100,
'message' => $message ?? 'Import berhasil diselesaikan',
'completed_at' => now(),
'updated_at' => now()
]);
Cache::put($cacheKey, $progressData, $this->cacheTtl);
Log::info('ImportProgressService: Import completed', [
'import_id' => $importId,
'total_rows' => $progressData['total_rows'],
'processed_rows' => $progressData['processed_rows'],
'skipped_rows' => $progressData['skipped_rows'],
'error_rows' => $progressData['error_rows']
]);
return $progressData;
}
/**
* Mark import as failed
*
* @param string $importId
* @param string $errorMessage
* @return array
*/
public function fail(string $importId, string $errorMessage): array
{
$cacheKey = $this->getCacheKey($importId);
$progressData = Cache::get($cacheKey);
if (!$progressData) {
Log::warning('ImportProgressService: Progress data not found for failure', ['import_id' => $importId]);
return [];
}
$progressData = array_merge($progressData, [
'status' => 'failed',
'message' => 'Import gagal: ' . $errorMessage,
'failed_at' => now(),
'updated_at' => now()
]);
Cache::put($cacheKey, $progressData, $this->cacheTtl);
Log::error('ImportProgressService: Import failed', [
'import_id' => $importId,
'error' => $errorMessage,
'progress_data' => $progressData
]);
return $progressData;
}
/**
* Get progress data
*
* @param string $importId
* @return array|null
*/
public function getProgress(string $importId): ?array
{
$cacheKey = $this->getCacheKey($importId);
return Cache::get($cacheKey);
}
/**
* Get all active imports for user
*
* @param int $userId
* @return array
*/
public function getUserImports(int $userId): array
{
$pattern = $this->cacheKeyPrefix . '_*';
$keys = Cache::get($pattern);
$imports = [];
foreach ($keys as $key) {
$data = Cache::get($key);
if ($data && $data['user_id'] === $userId) {
$imports[] = $data;
}
}
return $imports;
}
/**
* Clear progress data
*
* @param string $importId
* @return bool
*/
public function clear(string $importId): bool
{
$cacheKey = $this->getCacheKey($importId);
$result = Cache::forget($cacheKey);
Log::info('ImportProgressService: Progress data cleared', ['import_id' => $importId]);
return $result;
}
/**
* Generate cache key
*
* @param string $importId
* @return string
*/
private function getCacheKey(string $importId): string
{
return $this->cacheKeyPrefix . '_' . $importId;
}
}