✨ feat(slik): implementasi sistem import SLIK dengan optimasi memory & timeout handling
- Menambahkan `SlikController.php` dengan method CRUD dan import data SLIK, termasuk logging detail & error handling - Menambahkan `SlikImport.php` dengan Laravel Excel (ToCollection, WithChunkReading, WithBatchInserts, dll.) - Optimasi memory dengan chunk processing (50 baris/chunk) dan batch insert (50 record/batch) - Penanganan timeout menggunakan `set_time_limit` & memory limit configurable via config - Implementasi queue processing untuk file besar (>5MB) dengan progress tracking - Validasi file upload & data baris, skip header dari baris ke-5, serta rollback jika error - Garbage collection otomatis setiap 25 baris, unset variabel tidak terpakai, dan logging usage memory - Error handling komprehensif dengan try-catch, rollback transaksi, hapus file temp, dan logging stack trace - Semua parameter (batch size, chunk size, memory limit, timeout, GC, queue threshold) configurable via config - Diuji pada file besar (>50MB), memory stabil, timeout handling berfungsi, rollback aman, dan progress tracking valid - Catatan: pastikan queue worker berjalan, monitor log progress, sesuaikan config server, dan backup DB sebelum import
This commit is contained in:
Daeng Deni Mardaeni
236
app/Services/ImportProgressService.php
Normal file
236
app/Services/ImportProgressService.php
Normal file
@@ -0,0 +1,236 @@
|
||||
<?php
|
||||
|
||||
namespace Modules\Lpj\Services;
|
||||
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
|
||||
class ImportProgressService
|
||||
{
|
||||
protected string $cacheKeyPrefix;
|
||||
protected int $cacheTtl;
|
||||
|
||||
/**
|
||||
* Constructor
|
||||
*/
|
||||
public function __construct()
|
||||
{
|
||||
$this->cacheKeyPrefix = config('import.slik.progress.cache_key', 'slik_import_progress');
|
||||
$this->cacheTtl = config('import.slik.progress.cache_ttl', 3600);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start new import progress
|
||||
*
|
||||
* @param string $importId
|
||||
* @param int $userId
|
||||
* @param string $filename
|
||||
* @param int $totalRows
|
||||
* @return array
|
||||
*/
|
||||
public function start(string $importId, int $userId, string $filename, int $totalRows): array
|
||||
{
|
||||
$progressData = [
|
||||
'import_id' => $importId,
|
||||
'user_id' => $userId,
|
||||
'filename' => $filename,
|
||||
'total_rows' => $totalRows,
|
||||
'processed_rows' => 0,
|
||||
'skipped_rows' => 0,
|
||||
'error_rows' => 0,
|
||||
'status' => 'started',
|
||||
'percentage' => 0,
|
||||
'message' => 'Memulai import...',
|
||||
'started_at' => now(),
|
||||
'updated_at' => now()
|
||||
];
|
||||
|
||||
$cacheKey = $this->getCacheKey($importId);
|
||||
Cache::put($cacheKey, $progressData, $this->cacheTtl);
|
||||
|
||||
Log::info('ImportProgressService: Import started', $progressData);
|
||||
|
||||
return $progressData;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update progress import
|
||||
*
|
||||
* @param string $importId
|
||||
* @param int $processedRows
|
||||
* @param int $skippedRows
|
||||
* @param int $errorRows
|
||||
* @param string|null $message
|
||||
* @return array
|
||||
*/
|
||||
public function update(string $importId, int $processedRows, int $skippedRows, int $errorRows, ?string $message = null): array
|
||||
{
|
||||
$cacheKey = $this->getCacheKey($importId);
|
||||
$progressData = Cache::get($cacheKey);
|
||||
|
||||
if (!$progressData) {
|
||||
Log::warning('ImportProgressService: Progress data not found', ['import_id' => $importId]);
|
||||
return [];
|
||||
}
|
||||
|
||||
$totalRows = $progressData['total_rows'];
|
||||
$percentage = $totalRows > 0 ? round(($processedRows / $totalRows) * 100, 2) : 0;
|
||||
|
||||
$progressData = array_merge($progressData, [
|
||||
'processed_rows' => $processedRows,
|
||||
'skipped_rows' => $skippedRows,
|
||||
'error_rows' => $errorRows,
|
||||
'percentage' => $percentage,
|
||||
'message' => $message ?? "Memproses baris {$processedRows} dari {$totalRows}...",
|
||||
'updated_at' => now()
|
||||
]);
|
||||
|
||||
Cache::put($cacheKey, $progressData, $this->cacheTtl);
|
||||
|
||||
// Log progress setiap 10%
|
||||
if ($percentage % 10 === 0) {
|
||||
Log::info('ImportProgressService: Progress update', [
|
||||
'import_id' => $importId,
|
||||
'percentage' => $percentage,
|
||||
'processed' => $processedRows,
|
||||
'total' => $totalRows
|
||||
]);
|
||||
}
|
||||
|
||||
return $progressData;
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark import as completed
|
||||
*
|
||||
* @param string $importId
|
||||
* @param string|null $message
|
||||
* @return array
|
||||
*/
|
||||
public function complete(string $importId, ?string $message = null): array
|
||||
{
|
||||
$cacheKey = $this->getCacheKey($importId);
|
||||
$progressData = Cache::get($cacheKey);
|
||||
|
||||
if (!$progressData) {
|
||||
Log::warning('ImportProgressService: Progress data not found for completion', ['import_id' => $importId]);
|
||||
return [];
|
||||
}
|
||||
|
||||
$progressData = array_merge($progressData, [
|
||||
'status' => 'completed',
|
||||
'percentage' => 100,
|
||||
'message' => $message ?? 'Import berhasil diselesaikan',
|
||||
'completed_at' => now(),
|
||||
'updated_at' => now()
|
||||
]);
|
||||
|
||||
Cache::put($cacheKey, $progressData, $this->cacheTtl);
|
||||
|
||||
Log::info('ImportProgressService: Import completed', [
|
||||
'import_id' => $importId,
|
||||
'total_rows' => $progressData['total_rows'],
|
||||
'processed_rows' => $progressData['processed_rows'],
|
||||
'skipped_rows' => $progressData['skipped_rows'],
|
||||
'error_rows' => $progressData['error_rows']
|
||||
]);
|
||||
|
||||
return $progressData;
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark import as failed
|
||||
*
|
||||
* @param string $importId
|
||||
* @param string $errorMessage
|
||||
* @return array
|
||||
*/
|
||||
public function fail(string $importId, string $errorMessage): array
|
||||
{
|
||||
$cacheKey = $this->getCacheKey($importId);
|
||||
$progressData = Cache::get($cacheKey);
|
||||
|
||||
if (!$progressData) {
|
||||
Log::warning('ImportProgressService: Progress data not found for failure', ['import_id' => $importId]);
|
||||
return [];
|
||||
}
|
||||
|
||||
$progressData = array_merge($progressData, [
|
||||
'status' => 'failed',
|
||||
'message' => 'Import gagal: ' . $errorMessage,
|
||||
'failed_at' => now(),
|
||||
'updated_at' => now()
|
||||
]);
|
||||
|
||||
Cache::put($cacheKey, $progressData, $this->cacheTtl);
|
||||
|
||||
Log::error('ImportProgressService: Import failed', [
|
||||
'import_id' => $importId,
|
||||
'error' => $errorMessage,
|
||||
'progress_data' => $progressData
|
||||
]);
|
||||
|
||||
return $progressData;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get progress data
|
||||
*
|
||||
* @param string $importId
|
||||
* @return array|null
|
||||
*/
|
||||
public function getProgress(string $importId): ?array
|
||||
{
|
||||
$cacheKey = $this->getCacheKey($importId);
|
||||
return Cache::get($cacheKey);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all active imports for user
|
||||
*
|
||||
* @param int $userId
|
||||
* @return array
|
||||
*/
|
||||
public function getUserImports(int $userId): array
|
||||
{
|
||||
$pattern = $this->cacheKeyPrefix . '_*';
|
||||
$keys = Cache::get($pattern);
|
||||
|
||||
$imports = [];
|
||||
foreach ($keys as $key) {
|
||||
$data = Cache::get($key);
|
||||
if ($data && $data['user_id'] === $userId) {
|
||||
$imports[] = $data;
|
||||
}
|
||||
}
|
||||
|
||||
return $imports;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear progress data
|
||||
*
|
||||
* @param string $importId
|
||||
* @return bool
|
||||
*/
|
||||
public function clear(string $importId): bool
|
||||
{
|
||||
$cacheKey = $this->getCacheKey($importId);
|
||||
$result = Cache::forget($cacheKey);
|
||||
|
||||
Log::info('ImportProgressService: Progress data cleared', ['import_id' => $importId]);
|
||||
|
||||
return $result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate cache key
|
||||
*
|
||||
* @param string $importId
|
||||
* @return string
|
||||
*/
|
||||
private function getCacheKey(string $importId): string
|
||||
{
|
||||
return $this->cacheKeyPrefix . '_' . $importId;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user