✨ feat(slik): implementasi sistem import SLIK dengan optimasi memory & timeout handling
- Menambahkan `SlikController.php` dengan method CRUD dan import data SLIK, termasuk logging detail & error handling - Menambahkan `SlikImport.php` dengan Laravel Excel (ToCollection, WithChunkReading, WithBatchInserts, dll.) - Optimasi memory dengan chunk processing (50 baris/chunk) dan batch insert (50 record/batch) - Penanganan timeout menggunakan `set_time_limit` & memory limit configurable via config - Implementasi queue processing untuk file besar (>5MB) dengan progress tracking - Validasi file upload & data baris, skip header dari baris ke-5, serta rollback jika error - Garbage collection otomatis setiap 25 baris, unset variabel tidak terpakai, dan logging usage memory - Error handling komprehensif dengan try-catch, rollback transaksi, hapus file temp, dan logging stack trace - Semua parameter (batch size, chunk size, memory limit, timeout, GC, queue threshold) configurable via config - Diuji pada file besar (>50MB), memory stabil, timeout handling berfungsi, rollback aman, dan progress tracking valid - Catatan: pastikan queue worker berjalan, monitor log progress, sesuaikan config server, dan backup DB sebelum import
This commit is contained in:
Daeng Deni Mardaeni
179
app/Jobs/ProcessSlikImport.php
Normal file
179
app/Jobs/ProcessSlikImport.php
Normal file
@@ -0,0 +1,179 @@
|
||||
<?php
|
||||
|
||||
namespace Modules\Lpj\Jobs;
|
||||
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use Modules\Lpj\Imports\SlikImport;
|
||||
use Maatwebsite\Excel\Facades\Excel;
|
||||
|
||||
class ProcessSlikImport implements ShouldQueue
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
public $timeout = 1800; // 30 menit untuk file besar
|
||||
public $tries = 5; // Tambah retry untuk file sangat besar
|
||||
public $maxExceptions = 5;
|
||||
public $backoff = [60, 300, 900, 1800, 3600]; // Exponential backoff dalam detik
|
||||
|
||||
protected string $filePath;
|
||||
protected int $userId;
|
||||
protected string $importId;
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
*
|
||||
* @param string $filePath
|
||||
* @param int $userId
|
||||
* @param string $importId
|
||||
*/
|
||||
public function __construct(string $filePath, int $userId, string $importId)
|
||||
{
|
||||
$this->filePath = $filePath;
|
||||
$this->userId = $userId;
|
||||
$this->importId = $importId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the job.
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function handle(): void
|
||||
{
|
||||
Log::info('ProcessSlikImport: Memulai proses import via queue', [
|
||||
'file_path' => $this->filePath,
|
||||
'user_id' => $this->userId,
|
||||
'import_id' => $this->importId,
|
||||
'memory_limit' => ini_get('memory_limit'),
|
||||
'max_execution_time' => ini_get('max_execution_time')
|
||||
]);
|
||||
|
||||
try {
|
||||
// Cek file size terlebih dahulu
|
||||
$fileSize = filesize($this->filePath);
|
||||
$maxFileSize = config('import.slik.max_file_size', 50) * 1024 * 1024; // Convert MB to bytes
|
||||
|
||||
if ($fileSize > $maxFileSize) {
|
||||
throw new \Exception('File terlalu besar: ' . number_format($fileSize / 1024 / 1024, 2) . ' MB. Maksimum: ' . config('import.slik.max_file_size', 50) . ' MB');
|
||||
}
|
||||
|
||||
// Set optimasi memory untuk queue processing
|
||||
$memoryLimit = config('import.slik.memory_limit', 1024);
|
||||
ini_set('memory_limit', $memoryLimit . 'M');
|
||||
ini_set('max_execution_time', config('import.slik.timeout', 1800));
|
||||
|
||||
// Set timeout untuk XML Scanner
|
||||
$xmlScannerTimeout = config('import.slik.xml_scanner.timeout', 1800);
|
||||
$xmlScannerMemory = config('import.slik.xml_scanner.memory_limit', 1024);
|
||||
|
||||
// Enable garbage collection jika diizinkan
|
||||
if (config('import.slik.enable_gc', true)) {
|
||||
gc_enable();
|
||||
}
|
||||
|
||||
// Update progress status
|
||||
$this->updateProgress('processing', 0, 'Memproses file Excel...');
|
||||
|
||||
Log::info('SlikImport: Processing file', [
|
||||
'file' => basename($this->filePath),
|
||||
'file_size' => number_format(filesize($this->filePath) / 1024 / 1024, 2) . ' MB',
|
||||
'memory_limit' => $memoryLimit . 'M',
|
||||
'timeout' => config('import.slik.timeout', 1800),
|
||||
'enable_gc' => config('import.slik.enable_gc', true),
|
||||
'xml_scanner_timeout' => config('import.slik.xml_scanner.timeout', 1800),
|
||||
'chunk_size' => config('import.slik.chunk_size', 50),
|
||||
'batch_size' => config('import.slik.batch_size', 50),
|
||||
]);
|
||||
|
||||
// Import file menggunakan SlikImport
|
||||
$import = new SlikImport();
|
||||
Excel::import($import, $this->filePath);
|
||||
|
||||
// Update progress selesai
|
||||
$this->updateProgress('completed', 100, 'Import berhasil diselesaikan');
|
||||
|
||||
Log::info('ProcessSlikImport: Import berhasil diselesaikan', [
|
||||
'import_id' => $this->importId,
|
||||
'file_path' => $this->filePath,
|
||||
'memory_usage' => memory_get_usage(true),
|
||||
'memory_peak' => memory_get_peak_usage(true)
|
||||
]);
|
||||
|
||||
// Hapus file temporary setelah selesai
|
||||
if (config('import.general.cleanup_temp_files', true)) {
|
||||
Storage::delete($this->filePath);
|
||||
Log::info('ProcessSlikImport: File temporary dihapus', ['file_path' => $this->filePath]);
|
||||
}
|
||||
|
||||
} catch (\Exception $e) {
|
||||
// Update progress error
|
||||
$this->updateProgress('failed', 0, 'Error: ' . $e->getMessage());
|
||||
|
||||
Log::error('ProcessSlikImport: Error saat proses import', [
|
||||
'import_id' => $this->importId,
|
||||
'error' => $e->getMessage(),
|
||||
'trace' => $e->getTraceAsString(),
|
||||
'file' => $e->getFile(),
|
||||
'line' => $e->getLine(),
|
||||
'memory_usage' => memory_get_usage(true)
|
||||
]);
|
||||
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update progress import
|
||||
*
|
||||
* @param string $status
|
||||
* @param int $percentage
|
||||
* @param string $message
|
||||
* @return void
|
||||
*/
|
||||
private function updateProgress(string $status, int $percentage, string $message): void
|
||||
{
|
||||
if (config('import.slik.progress.enabled', true)) {
|
||||
$cacheKey = config('import.slik.progress.cache_key', 'slik_import_progress') . '_' . $this->importId;
|
||||
$cacheTtl = config('import.slik.progress.cache_ttl', 3600);
|
||||
|
||||
$progressData = [
|
||||
'status' => $status,
|
||||
'percentage' => $percentage,
|
||||
'message' => $message,
|
||||
'timestamp' => now(),
|
||||
'user_id' => $this->userId
|
||||
];
|
||||
|
||||
cache()->put($cacheKey, $progressData, $cacheTtl);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle job failure
|
||||
*
|
||||
* @param \Throwable $exception
|
||||
* @return void
|
||||
*/
|
||||
public function failed(\Throwable $exception): void
|
||||
{
|
||||
Log::error('ProcessSlikImport: Job failed', [
|
||||
'import_id' => $this->importId,
|
||||
'error' => $exception->getMessage(),
|
||||
'trace' => $exception->getTraceAsString()
|
||||
]);
|
||||
|
||||
// Update progress ke failed
|
||||
$this->updateProgress('failed', 0, 'Import gagal: ' . $exception->getMessage());
|
||||
|
||||
// Cleanup file temporary
|
||||
if (Storage::exists($this->filePath)) {
|
||||
Storage::delete($this->filePath);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user