✨ feat(slik): implementasi sistem import SLIK dengan optimasi memory & timeout handling
- Menambahkan `SlikController.php` dengan method CRUD dan import data SLIK, termasuk logging detail & error handling - Menambahkan `SlikImport.php` dengan Laravel Excel (ToCollection, WithChunkReading, WithBatchInserts, dll.) - Optimasi memory dengan chunk processing (50 baris/chunk) dan batch insert (50 record/batch) - Penanganan timeout menggunakan `set_time_limit` & memory limit configurable via config - Implementasi queue processing untuk file besar (>5MB) dengan progress tracking - Validasi file upload & data baris, skip header dari baris ke-5, serta rollback jika error - Garbage collection otomatis setiap 25 baris, unset variabel tidak terpakai, dan logging usage memory - Error handling komprehensif dengan try-catch, rollback transaksi, hapus file temp, dan logging stack trace - Semua parameter (batch size, chunk size, memory limit, timeout, GC, queue threshold) configurable via config - Diuji pada file besar (>50MB), memory stabil, timeout handling berfungsi, rollback aman, dan progress tracking valid - Catatan: pastikan queue worker berjalan, monitor log progress, sesuaikan config server, dan backup DB sebelum import
This commit is contained in:
Daeng Deni Mardaeni
473
app/Http/Controllers/SlikController.php
Normal file
473
app/Http/Controllers/SlikController.php
Normal file
@@ -0,0 +1,473 @@
|
||||
<?php
|
||||
|
||||
namespace Modules\Lpj\Http\Controllers;
|
||||
|
||||
use App\Http\Controllers\Controller;
|
||||
use Exception;
|
||||
use Illuminate\Http\Request;
|
||||
use Illuminate\Validation\ValidationException;
|
||||
use Maatwebsite\Excel\Facades\Excel;
|
||||
use Modules\Lpj\Imports\SlikImport;
|
||||
use Modules\Lpj\Models\Slik;
|
||||
use Illuminate\Support\Facades\Auth;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
|
||||
/**
|
||||
* Controller untuk mengelola data Slik
|
||||
*
|
||||
* Menangani operasi CRUD dan import data Slik dari file Excel
|
||||
* dengan fitur server-side processing untuk datatables
|
||||
*
|
||||
* @package Modules\Lpj\Http\Controllers
|
||||
*/
|
||||
class SlikController extends Controller
|
||||
{
|
||||
public $user;
|
||||
|
||||
/**
|
||||
* Constructor
|
||||
*/
|
||||
public function __construct()
|
||||
{
|
||||
$this->user = Auth::user();
|
||||
}
|
||||
|
||||
/**
|
||||
* Menampilkan halaman index slik
|
||||
*
|
||||
* @return \Illuminate\View\View
|
||||
*/
|
||||
public function index()
|
||||
{
|
||||
return view('lpj::slik.index');
|
||||
}
|
||||
|
||||
/**
|
||||
* Menampilkan detail slik
|
||||
*
|
||||
* @param int $id
|
||||
* @return \Illuminate\View\View
|
||||
*/
|
||||
public function show($id)
|
||||
{
|
||||
$slik = Slik::findOrFail($id);
|
||||
return view('lpj::slik.show', compact('slik'));
|
||||
}
|
||||
|
||||
/**
|
||||
* Data untuk datatables dengan server-side processing
|
||||
*
|
||||
* @param Request $request
|
||||
* @return \Illuminate\Http\JsonResponse
|
||||
*/
|
||||
public function dataForDatatables(Request $request)
|
||||
{
|
||||
// Authorization check dapat ditambahkan sesuai kebutuhan
|
||||
// if (is_null($this->user)) {
|
||||
// abort(403, 'Unauthorized access.');
|
||||
// }
|
||||
|
||||
// Retrieve data from the database
|
||||
$query = Slik::query();
|
||||
|
||||
// Apply search filter if provided
|
||||
if ($request->has('search') && !empty($request->get('search'))) {
|
||||
$search = $request->get('search');
|
||||
$query->where(function ($q) use ($search) {
|
||||
$q->where('sandi_bank', 'LIKE', "%$search%")
|
||||
->orWhere('no_rekening', 'LIKE', "%$search%")
|
||||
->orWhere('cif', 'LIKE', "%$search%")
|
||||
->orWhere('nama_debitur', 'LIKE', "%$search%")
|
||||
->orWhere('nama_cabang', 'LIKE', "%$search%")
|
||||
->orWhere('jenis_agunan', 'LIKE', "%$search%")
|
||||
->orWhere('nama_pemilik_agunan', 'LIKE', "%$search%")
|
||||
->orWhere('alamat_agunan', 'LIKE', "%$search%")
|
||||
->orWhere('lokasi_agunan', 'LIKE', "%$search%");
|
||||
});
|
||||
}
|
||||
|
||||
// Apply year filter
|
||||
if ($request->has('year') && !empty($request->get('year'))) {
|
||||
$query->byYear($request->get('year'));
|
||||
}
|
||||
|
||||
// Apply month filter
|
||||
if ($request->has('month') && !empty($request->get('month'))) {
|
||||
$query->byMonth($request->get('month'));
|
||||
}
|
||||
|
||||
// Apply sandi bank filter
|
||||
if ($request->has('sandi_bank') && !empty($request->get('sandi_bank'))) {
|
||||
$query->where('sandi_bank', $request->get('sandi_bank'));
|
||||
}
|
||||
|
||||
// Apply kolektibilitas filter
|
||||
if ($request->has('kolektibilitas') && !empty($request->get('kolektibilitas'))) {
|
||||
$query->where('kolektibilitas', $request->get('kolektibilitas'));
|
||||
}
|
||||
|
||||
// Apply jenis agunan filter
|
||||
if ($request->has('jenis_agunan') && !empty($request->get('jenis_agunan'))) {
|
||||
$query->where('jenis_agunan', $request->get('jenis_agunan'));
|
||||
}
|
||||
|
||||
// Apply sorting if provided
|
||||
if ($request->has('sortOrder') && !empty($request->get('sortOrder'))) {
|
||||
$order = $request->get('sortOrder');
|
||||
$column = $request->get('sortField', 'created_at');
|
||||
$query->orderBy($column, $order);
|
||||
} else {
|
||||
$query->orderBy('created_at', 'desc');
|
||||
}
|
||||
|
||||
// Get the total count of records
|
||||
$totalRecords = $query->count();
|
||||
|
||||
// Apply pagination if provided
|
||||
if ($request->has('page') && $request->has('size')) {
|
||||
$page = $request->get('page');
|
||||
$size = $request->get('size');
|
||||
$offset = ($page - 1) * $size; // Calculate the offset
|
||||
|
||||
$query->skip($offset)->take($size);
|
||||
}
|
||||
|
||||
// Get the filtered count of records
|
||||
$filteredRecords = $query->count();
|
||||
|
||||
// Get the data for the current page with relationships
|
||||
$data = $query->get();
|
||||
|
||||
// Transform data untuk datatables
|
||||
$transformedData = $data->map(function ($item) {
|
||||
return [
|
||||
'id' => $item->id,
|
||||
'sandi_bank' => $item->sandi_bank,
|
||||
'tahun' => $item->tahun,
|
||||
'bulan' => $item->bulan,
|
||||
'no_rekening' => $item->no_rekening,
|
||||
'cif' => $item->cif,
|
||||
'nama_debitur' => $item->nama_debitur,
|
||||
'kolektibilitas' => $item->kolektibilitas,
|
||||
'kolektibilitas_badge' => $item->kolektibilitas_badge,
|
||||
'fasilitas' => $item->fasilitas,
|
||||
'jenis_agunan' => $item->jenis_agunan,
|
||||
'nama_pemilik_agunan' => $item->nama_pemilik_agunan,
|
||||
'nilai_agunan' => $item->nilai_agunan_formatted,
|
||||
'nilai_agunan_ljk' => $item->nilai_agunan_ljk_formatted,
|
||||
'alamat_agunan' => $item->alamat_agunan,
|
||||
'lokasi_agunan' => $item->lokasi_agunan,
|
||||
'nama_cabang' => $item->nama_cabang,
|
||||
'kode_cabang' => $item->kode_cabang,
|
||||
'created_by' => $item->creator?->name ?? '-',
|
||||
'created_at' => dateFormat($item->created_at, true)
|
||||
];
|
||||
});
|
||||
|
||||
// Calculate the page count
|
||||
$pageCount = ceil($totalRecords / ($request->get('size', 10)));
|
||||
|
||||
// Calculate the current page number
|
||||
$currentPage = $request->get('page', 1);
|
||||
|
||||
// Return the response data as a JSON object
|
||||
return response()->json([
|
||||
'draw' => $request->get('draw'),
|
||||
'recordsTotal' => $totalRecords,
|
||||
'recordsFiltered' => $filteredRecords,
|
||||
'pageCount' => $pageCount,
|
||||
'page' => $currentPage,
|
||||
'totalCount' => $totalRecords,
|
||||
'data' => $transformedData,
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Import data slik dari Excel dengan optimasi memory dan progress tracking
|
||||
*
|
||||
* @param Request $request
|
||||
* @return \Illuminate\Http\RedirectResponse
|
||||
*/
|
||||
public function import(Request $request)
|
||||
{
|
||||
Log::info('SlikController: Starting import process with optimizations', [
|
||||
'user_id' => Auth::id(),
|
||||
'request_size' => $request->header('Content-Length'),
|
||||
'has_file' => $request->hasFile('file'),
|
||||
'memory_limit' => ini_get('memory_limit'),
|
||||
'max_execution_time' => ini_get('max_execution_time')
|
||||
]);
|
||||
|
||||
// Validasi file upload dengan logging detail dan error handling komprehensif
|
||||
try {
|
||||
// Cek apakah ada file yang diupload
|
||||
if (!$request->hasFile('file')) {
|
||||
Log::error('SlikController: Tidak ada file yang diupload', [
|
||||
'user_id' => Auth::id(),
|
||||
'files_count' => count($request->allFiles()),
|
||||
'request_data' => $request->all()
|
||||
]);
|
||||
throw ValidationException::withMessages(['file' => 'Tidak ada file yang diupload.']);
|
||||
}
|
||||
|
||||
$file = $request->file('file');
|
||||
|
||||
// Cek apakah file valid
|
||||
if (!$file->isValid()) {
|
||||
$error = $file->getError();
|
||||
$errorMessage = match($error) {
|
||||
UPLOAD_ERR_INI_SIZE => 'File terlalu besar (melebihi upload_max_filesize).',
|
||||
UPLOAD_ERR_FORM_SIZE => 'File terlalu besar (melebihi MAX_FILE_SIZE).',
|
||||
UPLOAD_ERR_PARTIAL => 'File hanya terupload sebagian.',
|
||||
UPLOAD_ERR_NO_FILE => 'Tidak ada file yang diupload.',
|
||||
UPLOAD_ERR_NO_TMP_DIR => 'Direktori temp tidak tersedia.',
|
||||
UPLOAD_ERR_CANT_WRITE => 'Gagal menulis file ke disk.',
|
||||
UPLOAD_ERR_EXTENSION => 'Upload dibatalkan oleh ekstensi PHP.',
|
||||
default => 'Error upload tidak diketahui: ' . $error
|
||||
};
|
||||
|
||||
Log::error('SlikController: File upload tidak valid', [
|
||||
'error' => $error,
|
||||
'error_message' => $errorMessage,
|
||||
'user_id' => Auth::id(),
|
||||
'file_info' => [
|
||||
'name' => $file->getClientOriginalName(),
|
||||
'size' => $file->getSize(),
|
||||
'mime' => $file->getMimeType()
|
||||
]
|
||||
]);
|
||||
throw ValidationException::withMessages(['file' => $errorMessage]);
|
||||
}
|
||||
|
||||
$maxFileSize = config('import.slik.max_file_size', 50) * 1024; // dalam KB
|
||||
$request->validate([
|
||||
'file' => 'required|file|mimes:xlsx,xls|max:' . $maxFileSize
|
||||
]);
|
||||
Log::info('SlikController: Validasi file berhasil');
|
||||
} catch (\Illuminate\Validation\ValidationException $e) {
|
||||
Log::error('SlikController: Validasi file gagal', [
|
||||
'errors' => $e->errors(),
|
||||
'user_id' => Auth::id(),
|
||||
'request_size' => $request->header('Content-Length')
|
||||
]);
|
||||
throw $e;
|
||||
}
|
||||
|
||||
try {
|
||||
$uploadedFile = $request->file('file');
|
||||
$originalName = $uploadedFile->getClientOriginalName();
|
||||
$fileSize = $uploadedFile->getSize();
|
||||
|
||||
Log::info('SlikController: Memulai import data Slik', [
|
||||
'user_id' => Auth::id(),
|
||||
'filename' => $originalName,
|
||||
'filesize' => $fileSize,
|
||||
'filesize_mb' => round($fileSize / 1024 / 1024, 2),
|
||||
'mime_type' => $uploadedFile->getMimeType(),
|
||||
'extension' => $uploadedFile->getClientOriginalExtension()
|
||||
]);
|
||||
|
||||
// Generate unique import ID
|
||||
$importId = uniqid('slik_import_');
|
||||
$userId = Auth::id() ?? 1;
|
||||
|
||||
// Cek apakah menggunakan queue processing untuk file besar
|
||||
$useQueue = config('import.slik.queue.enabled', false) && $fileSize > (5 * 1024 * 1024); // > 5MB
|
||||
|
||||
// Pastikan direktori temp ada
|
||||
$tempDir = storage_path('app/temp');
|
||||
if (!file_exists($tempDir)) {
|
||||
mkdir($tempDir, 0755, true);
|
||||
Log::info('SlikController: Direktori temp dibuat', ['path' => $tempDir]);
|
||||
}
|
||||
|
||||
// Simpan file sementara dengan nama unik
|
||||
$tempFileName = 'slik_import_' . time() . '_' . uniqid() . '.' . $uploadedFile->getClientOriginalExtension();
|
||||
$tempFilePath = $tempDir . '/' . $tempFileName;
|
||||
|
||||
Log::info('SlikController: Memindahkan file ke temp', [
|
||||
'temp_path' => $tempFilePath,
|
||||
'use_queue' => $useQueue
|
||||
]);
|
||||
|
||||
// Pindahkan file ke direktori temp
|
||||
$uploadedFile->move($tempDir, $tempFilePath);
|
||||
|
||||
// Verifikasi file berhasil dipindahkan
|
||||
if (!file_exists($tempFilePath)) {
|
||||
throw new Exception('File gagal dipindahkan ke direktori temp');
|
||||
}
|
||||
|
||||
Log::info('SlikController: File berhasil dipindahkan', [
|
||||
'file_size' => filesize($tempFilePath)
|
||||
]);
|
||||
|
||||
if ($useQueue) {
|
||||
Log::info('SlikController: Menggunakan queue processing untuk file besar', [
|
||||
'import_id' => $importId,
|
||||
'file_size_mb' => round($fileSize / 1024 / 1024, 2)
|
||||
]);
|
||||
|
||||
// Dispatch job ke queue
|
||||
\Modules\Lpj\Jobs\ProcessSlikImport::dispatch($tempFilePath, $userId, $importId);
|
||||
|
||||
return redirect()->back()->with('success', 'Import sedang diproses di background. ID: ' . $importId);
|
||||
}
|
||||
|
||||
// Import langsung untuk file kecil
|
||||
Log::info('SlikController: Processing file directly', [
|
||||
'import_id' => $importId,
|
||||
'file_size_mb' => round($fileSize / 1024 / 1024, 2)
|
||||
]);
|
||||
|
||||
// Set optimasi memory untuk import langsung
|
||||
$memoryLimit = config('import.slik.memory_limit', 256);
|
||||
ini_set('memory_limit', $memoryLimit . 'M');
|
||||
ini_set('max_execution_time', config('import.slik.timeout', 30000));
|
||||
|
||||
// Enable garbage collection jika diizinkan
|
||||
if (config('import.slik.enable_gc', true)) {
|
||||
gc_enable();
|
||||
}
|
||||
|
||||
// Proses import menggunakan SlikImport class
|
||||
Log::info('SlikController: Memulai proses Excel import');
|
||||
$import = new SlikImport();
|
||||
Excel::import($import, $tempFilePath);
|
||||
Log::info('SlikController: Excel import selesai');
|
||||
|
||||
// Force garbage collection setelah selesai
|
||||
if (config('import.slik.enable_gc', true)) {
|
||||
gc_collect_cycles();
|
||||
}
|
||||
|
||||
// Hapus file temporary setelah import
|
||||
if (file_exists($tempFilePath)) {
|
||||
unlink($tempFilePath);
|
||||
Log::info('SlikController: File temp berhasil dihapus');
|
||||
}
|
||||
|
||||
Log::info('SlikController: Data Slik berhasil diimport', [
|
||||
'user_id' => Auth::id(),
|
||||
'import_id' => $importId
|
||||
]);
|
||||
|
||||
return redirect()->back()->with('success', 'Data Slik berhasil diimport dari file Excel.');
|
||||
|
||||
} catch (Exception $e) {
|
||||
// Hapus file temporary jika ada error
|
||||
if (isset($tempFilePath) && file_exists($tempFilePath)) {
|
||||
unlink($tempFilePath);
|
||||
Log::info('SlikController: File temp dihapus karena error');
|
||||
}
|
||||
|
||||
Log::error('SlikController: Gagal import data Slik', [
|
||||
'error' => $e->getMessage(),
|
||||
'trace' => $e->getTraceAsString(),
|
||||
'user_id' => Auth::id(),
|
||||
'file' => $e->getFile(),
|
||||
'line' => $e->getLine(),
|
||||
'memory_usage' => memory_get_usage(true)
|
||||
]);
|
||||
|
||||
return redirect()->back()->with('error', 'Gagal import data Slik: ' . $e->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Menampilkan halaman form import
|
||||
*
|
||||
* @return \Illuminate\View\View
|
||||
*/
|
||||
public function importForm()
|
||||
{
|
||||
return view('lpj::slik.import');
|
||||
}
|
||||
|
||||
/**
|
||||
* Download template Excel untuk import
|
||||
*
|
||||
* @return \Symfony\Component\HttpFoundation\BinaryFileResponse
|
||||
*/
|
||||
public function downloadTemplate()
|
||||
{
|
||||
$templatePath = resource_path('metronic/slik.xlsx');
|
||||
|
||||
if (!file_exists($templatePath)) {
|
||||
return redirect()->back()->with('error', 'Template file tidak ditemukan.');
|
||||
}
|
||||
|
||||
return response()->download($templatePath, 'template_slik.xlsx');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get import progress
|
||||
*
|
||||
* @param string $importId
|
||||
* @return \Illuminate\Http\JsonResponse
|
||||
*/
|
||||
public function progress(string $importId)
|
||||
{
|
||||
try {
|
||||
$progressService = new \Modules\Lpj\Services\ImportProgressService();
|
||||
$progress = $progressService->getProgress($importId);
|
||||
|
||||
if (!$progress) {
|
||||
return response()->json([
|
||||
'success' => false,
|
||||
'message' => 'Progress import tidak ditemukan'
|
||||
], 404);
|
||||
}
|
||||
|
||||
return response()->json([
|
||||
'success' => true,
|
||||
'progress' => $progress
|
||||
]);
|
||||
|
||||
} catch (\Exception $e) {
|
||||
Log::error('SlikController: Error getting progress', [
|
||||
'import_id' => $importId,
|
||||
'error' => $e->getMessage()
|
||||
]);
|
||||
|
||||
return response()->json([
|
||||
'success' => false,
|
||||
'message' => 'Gagal mendapatkan progress: ' . $e->getMessage()
|
||||
], 500);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Hapus semua data slik
|
||||
*
|
||||
* @return \Illuminate\Http\RedirectResponse
|
||||
*/
|
||||
public function truncate()
|
||||
{
|
||||
DB::beginTransaction();
|
||||
try {
|
||||
Log::info('SlikController: Menghapus semua data Slik', [
|
||||
'user_id' => Auth::id()
|
||||
]);
|
||||
|
||||
Slik::truncate();
|
||||
|
||||
DB::commit();
|
||||
Log::info('SlikController: Semua data Slik berhasil dihapus', [
|
||||
'user_id' => Auth::id()
|
||||
]);
|
||||
|
||||
return redirect()->back()->with('success', 'Semua data Slik berhasil dihapus.');
|
||||
|
||||
} catch (Exception $e) {
|
||||
DB::rollback();
|
||||
Log::error('SlikController: Gagal menghapus data Slik', [
|
||||
'error' => $e->getMessage(),
|
||||
'user_id' => Auth::id()
|
||||
]);
|
||||
|
||||
return redirect()->back()->with('error', 'Gagal menghapus data Slik: ' . $e->getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user