feat(webstatement): tambah ProcessProvinceDataJob untuk import data provinsi
Perubahan yang dilakukan: - Membuat job baru ProcessProvinceDataJob dengan referensi dari ProcessSectorDataJob. - Menggunakan model ProvinceCore untuk menyimpan data provinsi. - Mendukung format file ST.PROVINCE.csv dengan delimiter khusus tilde (~). - Menambahkan validasi untuk kolom: id, date_time, province, dan province_name. - Mengabaikan baris header pada file saat proses import. - Menggunakan database transaction untuk menjaga konsistensi data. - Menambahkan counter untuk memantau jumlah record yang dilewati (skipped). - Mengimplementasikan error handling dan logging yang detail. - Menggunakan updateOrCreate untuk mencegah duplikasi data. - Menambahkan method failed() untuk menangani kasus job failure. - Melakukan mapping field province ke code dan province_name ke name. - Melakukan validasi data wajib sebelum menyimpan ke database. Tujuan perubahan: - Memfasilitasi proses import data provinsi dari file eksternal secara otomatis dan aman. - Menjamin data yang masuk telah tervalidasi dan bebas duplikasi. - Menyediakan log dan feedback yang cukup saat terjadi kegagalan.
This commit is contained in:
Daeng Deni Mardaeni
@@ -1,4 +1,5 @@
|
||||
<?php
|
||||
|
||||
namespace Modules\Webstatement\Http\Controllers;
|
||||
|
||||
use App\Http\Controllers\Controller;
|
||||
@@ -6,7 +7,7 @@
|
||||
use Exception;
|
||||
use Illuminate\Http\JsonResponse;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use Log;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Modules\Webstatement\Jobs\{ProcessAccountDataJob,
|
||||
ProcessArrangementDataJob,
|
||||
ProcessAtmTransactionJob,
|
||||
@@ -22,7 +23,8 @@
|
||||
ProcessStmtNarrParamDataJob,
|
||||
ProcessTellerDataJob,
|
||||
ProcessTransactionDataJob,
|
||||
ProcessSectorDataJob};
|
||||
ProcessSectorDataJob,
|
||||
ProcessProvinceDataJob};
|
||||
|
||||
class MigrasiController extends Controller
|
||||
{
|
||||
@@ -42,7 +44,8 @@
|
||||
'atmTransaction' => ProcessAtmTransactionJob::class,
|
||||
'arrangement' => ProcessArrangementDataJob::class,
|
||||
'billDetail' => ProcessBillDetailDataJob::class,
|
||||
'sector' => ProcessSectorDataJob::class
|
||||
'sector' => ProcessSectorDataJob::class,
|
||||
'province' => ProcessProvinceDataJob::class
|
||||
];
|
||||
|
||||
private const PARAMETER_PROCESSES = [
|
||||
@@ -50,7 +53,8 @@
|
||||
'stmtNarrParam',
|
||||
'stmtNarrFormat',
|
||||
'ftTxnTypeCondition',
|
||||
'sector'
|
||||
'sector',
|
||||
'province'
|
||||
];
|
||||
|
||||
private const DATA_PROCESSES = [
|
||||
|
||||
300
app/Jobs/ProcessProvinceDataJob.php
Normal file
300
app/Jobs/ProcessProvinceDataJob.php
Normal file
@@ -0,0 +1,300 @@
|
||||
<?php
|
||||
|
||||
namespace Modules\Webstatement\Jobs;
|
||||
|
||||
use Exception;
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Modules\Webstatement\Models\ProvinceCore;
|
||||
|
||||
class ProcessProvinceDataJob implements ShouldQueue
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
private const CSV_DELIMITER = '~';
|
||||
private const MAX_EXECUTION_TIME = 86400; // 24 hours in seconds
|
||||
private const FILENAME = 'ST.PROVINCE.csv';
|
||||
private const DISK_NAME = 'sftpStatement';
|
||||
|
||||
private string $period = '';
|
||||
private int $processedCount = 0;
|
||||
private int $errorCount = 0;
|
||||
private int $skippedCount = 0;
|
||||
|
||||
/**
|
||||
* Membuat instance job baru untuk memproses data provinsi
|
||||
*
|
||||
* @param string $period Periode data yang akan diproses
|
||||
*/
|
||||
public function __construct(string $period = '')
|
||||
{
|
||||
$this->period = $period;
|
||||
Log::info('ProcessProvinceDataJob: Job dibuat untuk periode: ' . $period);
|
||||
}
|
||||
|
||||
/**
|
||||
* Menjalankan job untuk memproses file ST.PROVINCE.csv
|
||||
* Menggunakan transaction untuk memastikan konsistensi data
|
||||
*
|
||||
* @return void
|
||||
* @throws Exception
|
||||
*/
|
||||
public function handle(): void
|
||||
{
|
||||
DB::beginTransaction();
|
||||
|
||||
try {
|
||||
Log::info('ProcessProvinceDataJob: Memulai pemrosesan data provinsi');
|
||||
|
||||
$this->initializeJob();
|
||||
|
||||
if ($this->period === '') {
|
||||
Log::warning('ProcessProvinceDataJob: Tidak ada periode yang diberikan untuk pemrosesan data provinsi');
|
||||
DB::rollback();
|
||||
return;
|
||||
}
|
||||
|
||||
$this->processPeriod();
|
||||
$this->logJobCompletion();
|
||||
|
||||
DB::commit();
|
||||
Log::info('ProcessProvinceDataJob: Transaction berhasil di-commit');
|
||||
|
||||
} catch (Exception $e) {
|
||||
DB::rollback();
|
||||
Log::error('ProcessProvinceDataJob: Error dalam pemrosesan, transaction di-rollback: ' . $e->getMessage());
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Inisialisasi pengaturan job
|
||||
* Mengatur timeout dan reset counter
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
private function initializeJob(): void
|
||||
{
|
||||
set_time_limit(self::MAX_EXECUTION_TIME);
|
||||
$this->processedCount = 0;
|
||||
$this->errorCount = 0;
|
||||
$this->skippedCount = 0;
|
||||
|
||||
Log::info('ProcessProvinceDataJob: Job diinisialisasi dengan timeout ' . self::MAX_EXECUTION_TIME . ' detik');
|
||||
}
|
||||
|
||||
/**
|
||||
* Memproses file untuk periode tertentu
|
||||
* Mengambil file dari SFTP dan memproses data
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
private function processPeriod(): void
|
||||
{
|
||||
$disk = Storage::disk(self::DISK_NAME);
|
||||
$filePath = "$this->period/" . self::FILENAME;
|
||||
|
||||
Log::info('ProcessProvinceDataJob: Memproses periode ' . $this->period);
|
||||
|
||||
if (!$this->validateFile($disk, $filePath)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$tempFilePath = $this->createTemporaryFile($disk, $filePath);
|
||||
$this->processFile($tempFilePath, $filePath);
|
||||
$this->cleanup($tempFilePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validasi keberadaan file di storage
|
||||
*
|
||||
* @param mixed $disk Storage disk instance
|
||||
* @param string $filePath Path file yang akan divalidasi
|
||||
* @return bool
|
||||
*/
|
||||
private function validateFile($disk, string $filePath): bool
|
||||
{
|
||||
Log::info("ProcessProvinceDataJob: Memvalidasi file provinsi: $filePath");
|
||||
|
||||
if (!$disk->exists($filePath)) {
|
||||
Log::warning("ProcessProvinceDataJob: File tidak ditemukan: $filePath");
|
||||
return false;
|
||||
}
|
||||
|
||||
Log::info("ProcessProvinceDataJob: File ditemukan dan valid: $filePath");
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Membuat file temporary untuk pemrosesan
|
||||
*
|
||||
* @param mixed $disk Storage disk instance
|
||||
* @param string $filePath Path file sumber
|
||||
* @return string Path file temporary
|
||||
*/
|
||||
private function createTemporaryFile($disk, string $filePath): string
|
||||
{
|
||||
$tempFilePath = storage_path("app/temp_" . self::FILENAME);
|
||||
file_put_contents($tempFilePath, $disk->get($filePath));
|
||||
|
||||
Log::info("ProcessProvinceDataJob: File temporary dibuat: $tempFilePath");
|
||||
return $tempFilePath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Memproses file CSV dan mengimpor data ke database
|
||||
* Format CSV: id~date_time~province~province_name
|
||||
*
|
||||
* @param string $tempFilePath Path file temporary
|
||||
* @param string $filePath Path file asli untuk logging
|
||||
* @return void
|
||||
*/
|
||||
private function processFile(string $tempFilePath, string $filePath): void
|
||||
{
|
||||
$handle = fopen($tempFilePath, "r");
|
||||
if ($handle === false) {
|
||||
Log::error("ProcessProvinceDataJob: Tidak dapat membuka file: $filePath");
|
||||
return;
|
||||
}
|
||||
|
||||
Log::info("ProcessProvinceDataJob: Memulai pemrosesan file: $filePath");
|
||||
|
||||
$rowCount = 0;
|
||||
$isFirstRow = true;
|
||||
|
||||
while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) {
|
||||
$rowCount++;
|
||||
|
||||
// Skip header row
|
||||
if ($isFirstRow) {
|
||||
$isFirstRow = false;
|
||||
Log::info("ProcessProvinceDataJob: Melewati header row: " . implode(self::CSV_DELIMITER, $row));
|
||||
continue;
|
||||
}
|
||||
|
||||
$this->processRow($row, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
Log::info("ProcessProvinceDataJob: Selesai memproses $filePath. Total baris: $rowCount, Diproses: {$this->processedCount}, Error: {$this->errorCount}, Dilewati: {$this->skippedCount}");
|
||||
}
|
||||
|
||||
/**
|
||||
* Memproses satu baris data CSV
|
||||
*
|
||||
* @param array $row Data baris CSV
|
||||
* @param int $rowCount Nomor baris untuk logging
|
||||
* @param string $filePath Path file untuk logging
|
||||
* @return void
|
||||
*/
|
||||
private function processRow(array $row, int $rowCount, string $filePath): void
|
||||
{
|
||||
// Validasi jumlah kolom (id~date_time~province~province_name = 4 kolom)
|
||||
if (count($row) !== 4) {
|
||||
Log::warning("ProcessProvinceDataJob: Baris $rowCount di $filePath memiliki jumlah kolom yang salah. Diharapkan: 4, Ditemukan: " . count($row));
|
||||
$this->skippedCount++;
|
||||
return;
|
||||
}
|
||||
|
||||
// Map data sesuai format CSV
|
||||
$data = [
|
||||
'code' => trim($row[2]), // province code
|
||||
'name' => trim($row[3]) // province_name
|
||||
];
|
||||
|
||||
Log::debug("ProcessProvinceDataJob: Memproses baris $rowCount dengan data: " . json_encode($data));
|
||||
|
||||
$this->saveRecord($data, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Menyimpan record provinsi ke database
|
||||
* Menggunakan updateOrCreate untuk menghindari duplikasi
|
||||
*
|
||||
* @param array $data Data provinsi yang akan disimpan
|
||||
* @param int $rowCount Nomor baris untuk logging
|
||||
* @param string $filePath Path file untuk logging
|
||||
* @return void
|
||||
*/
|
||||
private function saveRecord(array $data, int $rowCount, string $filePath): void
|
||||
{
|
||||
try {
|
||||
// Validasi data wajib
|
||||
if (empty($data['code']) || empty($data['name'])) {
|
||||
Log::warning("ProcessProvinceDataJob: Baris $rowCount di $filePath memiliki data kosong. Code: '{$data['code']}', Name: '{$data['name']}'");
|
||||
$this->skippedCount++;
|
||||
return;
|
||||
}
|
||||
|
||||
// Simpan atau update data provinsi
|
||||
$province = ProvinceCore::updateOrCreate(
|
||||
['code' => $data['code']], // Kondisi pencarian
|
||||
['name' => $data['name']] // Data yang akan diupdate/insert
|
||||
);
|
||||
|
||||
$this->processedCount++;
|
||||
Log::debug("ProcessProvinceDataJob: Berhasil menyimpan provinsi ID: {$province->id}, Code: {$data['code']}, Name: {$data['name']}");
|
||||
|
||||
} catch (Exception $e) {
|
||||
$this->errorCount++;
|
||||
Log::error("ProcessProvinceDataJob: Error menyimpan data provinsi pada baris $rowCount di $filePath: " . $e->getMessage());
|
||||
Log::error("ProcessProvinceDataJob: Data yang error: " . json_encode($data));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Membersihkan file temporary
|
||||
*
|
||||
* @param string $tempFilePath Path file temporary yang akan dihapus
|
||||
* @return void
|
||||
*/
|
||||
private function cleanup(string $tempFilePath): void
|
||||
{
|
||||
if (file_exists($tempFilePath)) {
|
||||
unlink($tempFilePath);
|
||||
Log::info("ProcessProvinceDataJob: File temporary dihapus: $tempFilePath");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Logging hasil akhir pemrosesan job
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
private function logJobCompletion(): void
|
||||
{
|
||||
$message = "ProcessProvinceDataJob: Pemrosesan data provinsi selesai. " .
|
||||
"Total diproses: {$this->processedCount}, " .
|
||||
"Total error: {$this->errorCount}, " .
|
||||
"Total dilewati: {$this->skippedCount}";
|
||||
|
||||
Log::info($message);
|
||||
|
||||
// Log summary untuk monitoring
|
||||
if ($this->errorCount > 0) {
|
||||
Log::warning("ProcessProvinceDataJob: Terdapat {$this->errorCount} error dalam pemrosesan");
|
||||
}
|
||||
|
||||
if ($this->skippedCount > 0) {
|
||||
Log::info("ProcessProvinceDataJob: Terdapat {$this->skippedCount} baris yang dilewati");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle job failure
|
||||
*
|
||||
* @param Exception $exception
|
||||
* @return void
|
||||
*/
|
||||
public function failed(Exception $exception): void
|
||||
{
|
||||
Log::error('ProcessProvinceDataJob: Job gagal dijalankan: ' . $exception->getMessage());
|
||||
Log::error('ProcessProvinceDataJob: Stack trace: ' . $exception->getTraceAsString());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user