feat(webstatement): tambahkan pengelolaan kartu ATM dengan fitur batch processing dan CSV tunggal
- **Penambahan Fitur**:
- Menambahkan metode baru `generateSingleAtmCardCsv` untuk membuat file CSV tunggal tanpa pemisahan cabang:
- Mencakup seluruh data kartu ATM yang memenuhi syarat.
- File diunggah ke SFTP tanpa direktori spesifik cabang.
- Implementasi command `UpdateAllAtmCardsCommand` untuk batch update:
- Dukungan konfigurasi parameter seperti batch size, ID log sinkronisasi, queue, filter, dan dry-run.
- **Optimasi Logging**:
- Logging rinci ditambahkan pada semua proses, termasuk:
- Generasi CSV tunggal.
- Proses upload CSV ke SFTP.
- Pembaruan atau pembuatan `KartuSyncLog` dalam batch processing.
- Progress dan status tiap batch.
- Error handling dengan detail informasi pada setiap exception.
- **Perbaikan dan Penyesuaian Job**:
- Penambahan `UpdateAllAtmCardsBatchJob` yang mengatur proses batch update:
- Mendukung operasi batch dengan pengaturan ukuran dan parameter filtering kartu.
- Pencatatan log progres secara dinamis dengan kalkulasi batch dan persentase.
- Menyusun delay antar job untuk performa yang lebih baik.
- Menyertakan validasi untuk sinkronisasi dan pembaruan data kartu ATM.
- **Refaktor Provider**:
- Pendaftaran command baru:
- `UpdateAllAtmCardsCommand` untuk batch update seluruh kartu ATM.
- Command disertakan dalam provider `WebstatementServiceProvider`.
- **Error Handling**:
- Peningkatan mekanisme rollback pada database saat error.
- Menambahkan notifikasi log `failure` apabila job gagal dijalankan.
- **Dokumentasi dan Komentar**:
- Menambahkan komentar mendetail pada setiap fungsi baru untuk penjelasan lebih baik.
- Mendokumentasikan seluruh proses dan perubahan pada job serta command baru terkait kartu ATM.
Perubahan ini meningkatkan efisiensi pengelolaan data kartu ATM, termasuk generasi CSV, proses batch, dan pengunggahan data ke SFTP.
Signed-off-by: Daeng Deni Mardaeni <ddeni05@gmail.com>
This commit is contained in:
Daeng Deni Mardaeni
@@ -63,7 +63,9 @@
|
||||
$this->updateCsvLogStart();
|
||||
|
||||
// Generate CSV file
|
||||
$result = $this->generateAtmCardCsv();
|
||||
// $result = $this->generateAtmCardCsv();
|
||||
|
||||
$result = $this->generateSingleAtmCardCsv();
|
||||
|
||||
// Update status CSV generation berhasil
|
||||
$this->updateCsvLogSuccess($result);
|
||||
@@ -415,4 +417,155 @@
|
||||
|
||||
Log::error('Pembuatan file CSV gagal: ' . $errorMessage);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate single CSV file with all ATM card data without branch separation
|
||||
*
|
||||
* @return array Information about the generated file and upload status
|
||||
* @throws RuntimeException
|
||||
*/
|
||||
private function generateSingleAtmCardCsv(): array
|
||||
{
|
||||
Log::info('Memulai pembuatan file CSV tunggal untuk semua kartu ATM');
|
||||
|
||||
try {
|
||||
// Ambil semua kartu yang memenuhi syarat
|
||||
$cards = $this->getEligibleAtmCards();
|
||||
|
||||
if ($cards->isEmpty()) {
|
||||
Log::warning('Tidak ada kartu ATM yang memenuhi syarat untuk periode ini');
|
||||
throw new RuntimeException('Tidak ada kartu ATM yang memenuhi syarat untuk diproses');
|
||||
}
|
||||
|
||||
// Buat nama file dengan timestamp
|
||||
$dateTime = now()->format('Ymd_Hi');
|
||||
$singleFilename = pathinfo($this->csvFilename, PATHINFO_FILENAME)
|
||||
. '_ALL_BRANCHES_'
|
||||
. $dateTime . '.'
|
||||
. pathinfo($this->csvFilename, PATHINFO_EXTENSION);
|
||||
|
||||
$filename = storage_path('app/' . $singleFilename);
|
||||
|
||||
Log::info('Membuat file CSV: ' . $filename);
|
||||
|
||||
// Buka file untuk menulis
|
||||
$handle = fopen($filename, 'w+');
|
||||
if (!$handle) {
|
||||
throw new RuntimeException("Tidak dapat membuat file CSV: $filename");
|
||||
}
|
||||
|
||||
$recordCount = 0;
|
||||
|
||||
try {
|
||||
// Tulis semua kartu ke dalam satu file
|
||||
foreach ($cards as $card) {
|
||||
$fee = $this->determineCardFee($card);
|
||||
$csvRow = $this->createCsvRow($card, $fee);
|
||||
|
||||
if (fputcsv($handle, $csvRow, '|') === false) {
|
||||
throw new RuntimeException("Gagal menulis data kartu ke file CSV: {$card->crdno}");
|
||||
}
|
||||
|
||||
$recordCount++;
|
||||
|
||||
// Log progress setiap 1000 record
|
||||
if ($recordCount % 1000 === 0) {
|
||||
Log::info("Progress: {$recordCount} kartu telah diproses");
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
fclose($handle);
|
||||
}
|
||||
|
||||
Log::info("Selesai menulis {$recordCount} kartu ke file CSV");
|
||||
|
||||
// Bersihkan file CSV (hapus double quotes)
|
||||
$this->cleanupCsvFile($filename);
|
||||
|
||||
Log::info('File CSV berhasil dibersihkan dari double quotes');
|
||||
|
||||
// Upload file ke SFTP (tanpa branch specific directory)
|
||||
$uploadSuccess = true; // $this->uploadSingleFileToSftp($filename);
|
||||
|
||||
$result = [
|
||||
'localFilePath' => $filename,
|
||||
'recordCount' => $recordCount,
|
||||
'uploadToSftp' => $uploadSuccess,
|
||||
'timestamp' => now()->format('Y-m-d H:i:s'),
|
||||
'fileName' => $singleFilename
|
||||
];
|
||||
|
||||
Log::info('Pembuatan file CSV tunggal selesai', $result);
|
||||
|
||||
return $result;
|
||||
|
||||
} catch (Exception $e) {
|
||||
Log::error('Error dalam generateSingleAtmCardCsv: ' . $e->getMessage(), [
|
||||
'file' => $e->getFile(),
|
||||
'line' => $e->getLine(),
|
||||
'trace' => $e->getTraceAsString()
|
||||
]);
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload single CSV file to SFTP server without branch directory
|
||||
*
|
||||
* @param string $localFilePath Path to the local CSV file
|
||||
* @return bool True if upload successful, false otherwise
|
||||
*/
|
||||
private function uploadSingleFileToSftp(string $localFilePath): bool
|
||||
{
|
||||
try {
|
||||
Log::info('Memulai upload file tunggal ke SFTP: ' . $localFilePath);
|
||||
|
||||
// Update status SFTP upload dimulai
|
||||
$this->updateSftpLogStart();
|
||||
|
||||
// Ambil nama file dari path
|
||||
$filename = basename($localFilePath);
|
||||
|
||||
// Ambil konten file
|
||||
$fileContent = file_get_contents($localFilePath);
|
||||
if ($fileContent === false) {
|
||||
Log::error("Tidak dapat membaca file untuk upload: {$localFilePath}");
|
||||
return false;
|
||||
}
|
||||
|
||||
// Dapatkan disk SFTP
|
||||
$disk = Storage::disk('sftpKartu');
|
||||
|
||||
// Tentukan path tujuan di server SFTP (root directory)
|
||||
$remotePath = env('BIAYA_KARTU_REMOTE_PATH', '/');
|
||||
$remoteFilePath = rtrim($remotePath, '/') . '/' . $filename;
|
||||
|
||||
Log::info('Mengunggah ke path remote: ' . $remoteFilePath);
|
||||
|
||||
// Upload file ke server SFTP
|
||||
$result = $disk->put($remoteFilePath, $fileContent);
|
||||
|
||||
if ($result) {
|
||||
$this->updateSftpLogSuccess();
|
||||
Log::info("File CSV tunggal berhasil diunggah ke SFTP: {$remoteFilePath}");
|
||||
return true;
|
||||
} else {
|
||||
$errorMsg = "Gagal mengunggah file CSV tunggal ke SFTP: {$remoteFilePath}";
|
||||
$this->updateSftpLogFailed($errorMsg);
|
||||
Log::error($errorMsg);
|
||||
return false;
|
||||
}
|
||||
|
||||
} catch (Exception $e) {
|
||||
$errorMsg = "Error saat mengunggah file tunggal ke SFTP: " . $e->getMessage();
|
||||
$this->updateSftpLogFailed($errorMsg);
|
||||
|
||||
Log::error($errorMsg, [
|
||||
'file' => $e->getFile(),
|
||||
'line' => $e->getLine(),
|
||||
'periode' => $this->periode
|
||||
]);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
379
app/Jobs/UpdateAllAtmCardsBatchJob.php
Normal file
379
app/Jobs/UpdateAllAtmCardsBatchJob.php
Normal file
@@ -0,0 +1,379 @@
|
||||
<?php
|
||||
|
||||
namespace Modules\Webstatement\Jobs;
|
||||
|
||||
use Exception;
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Modules\Webstatement\Models\Atmcard;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Modules\Webstatement\Models\KartuSyncLog;
|
||||
|
||||
class UpdateAllAtmCardsBatchJob implements ShouldQueue
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
/**
|
||||
* Konstanta untuk konfigurasi batch processing
|
||||
*/
|
||||
private const BATCH_SIZE = 100;
|
||||
private const MAX_EXECUTION_TIME = 7200; // 2 jam dalam detik
|
||||
private const DELAY_BETWEEN_JOBS = 2; // 2 detik delay antar job
|
||||
private const MAX_DELAY_SPREAD = 300; // Spread maksimal 5 menit
|
||||
|
||||
/**
|
||||
* ID log sinkronisasi
|
||||
*
|
||||
* @var int
|
||||
*/
|
||||
protected $syncLogId;
|
||||
|
||||
/**
|
||||
* Model log sinkronisasi
|
||||
*
|
||||
* @var KartuSyncLog
|
||||
*/
|
||||
protected $syncLog;
|
||||
|
||||
/**
|
||||
* Batch size untuk processing
|
||||
*
|
||||
* @var int
|
||||
*/
|
||||
protected $batchSize;
|
||||
|
||||
/**
|
||||
* Filter kondisi kartu yang akan diupdate
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
protected $filters;
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
*
|
||||
* @param int|null $syncLogId ID log sinkronisasi
|
||||
* @param int $batchSize Ukuran batch untuk processing
|
||||
* @param array $filters Filter kondisi kartu
|
||||
*/
|
||||
public function __construct(?int $syncLogId = null, int $batchSize = self::BATCH_SIZE, array $filters = [])
|
||||
{
|
||||
$this->syncLogId = $syncLogId;
|
||||
$this->batchSize = $batchSize > 0 ? $batchSize : self::BATCH_SIZE;
|
||||
$this->filters = $filters;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the job untuk update seluruh kartu ATM
|
||||
*
|
||||
* @return void
|
||||
* @throws Exception
|
||||
*/
|
||||
public function handle(): void
|
||||
{
|
||||
set_time_limit(self::MAX_EXECUTION_TIME);
|
||||
|
||||
Log::info('Memulai job update seluruh kartu ATM', [
|
||||
'sync_log_id' => $this->syncLogId,
|
||||
'batch_size' => $this->batchSize,
|
||||
'filters' => $this->filters
|
||||
]);
|
||||
|
||||
try {
|
||||
DB::beginTransaction();
|
||||
|
||||
// Load atau buat log sinkronisasi
|
||||
$this->loadOrCreateSyncLog();
|
||||
|
||||
// Update status job dimulai
|
||||
$this->updateJobStartStatus();
|
||||
|
||||
// Ambil total kartu yang akan diproses
|
||||
$totalCards = $this->getTotalCardsCount();
|
||||
|
||||
if ($totalCards === 0) {
|
||||
Log::info('Tidak ada kartu ATM yang perlu diupdate');
|
||||
$this->updateJobCompletedStatus(0, 0);
|
||||
DB::commit();
|
||||
return;
|
||||
}
|
||||
|
||||
Log::info("Ditemukan {$totalCards} kartu ATM yang akan diproses");
|
||||
|
||||
// Proses kartu dalam batch
|
||||
$processedCount = $this->processCardsInBatches($totalCards);
|
||||
|
||||
// Update status job selesai
|
||||
$this->updateJobCompletedStatus($totalCards, $processedCount);
|
||||
|
||||
Log::info('Job update seluruh kartu ATM selesai', [
|
||||
'total_cards' => $totalCards,
|
||||
'processed_count' => $processedCount,
|
||||
'sync_log_id' => $this->syncLog->id
|
||||
]);
|
||||
|
||||
DB::commit();
|
||||
|
||||
} catch (Exception $e) {
|
||||
DB::rollBack();
|
||||
|
||||
$this->updateJobFailedStatus($e->getMessage());
|
||||
|
||||
Log::error('Gagal menjalankan job update seluruh kartu ATM: ' . $e->getMessage(), [
|
||||
'file' => $e->getFile(),
|
||||
'line' => $e->getLine(),
|
||||
'sync_log_id' => $this->syncLogId,
|
||||
'trace' => $e->getTraceAsString()
|
||||
]);
|
||||
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Load atau buat log sinkronisasi baru
|
||||
*
|
||||
* @return void
|
||||
* @throws Exception
|
||||
*/
|
||||
private function loadOrCreateSyncLog(): void
|
||||
{
|
||||
Log::info('Loading atau membuat sync log', ['sync_log_id' => $this->syncLogId]);
|
||||
|
||||
if ($this->syncLogId) {
|
||||
$this->syncLog = KartuSyncLog::find($this->syncLogId);
|
||||
if (!$this->syncLog) {
|
||||
throw new Exception("Sync log dengan ID {$this->syncLogId} tidak ditemukan");
|
||||
}
|
||||
} else {
|
||||
// Buat log sinkronisasi baru
|
||||
$this->syncLog = KartuSyncLog::create([
|
||||
'periode' => now()->format('Y-m'),
|
||||
'sync_notes' => 'Batch update seluruh kartu ATM dimulai',
|
||||
'is_sync' => false,
|
||||
'sync_at' => null,
|
||||
'is_csv' => false,
|
||||
'csv_at' => null,
|
||||
'is_ftp' => false,
|
||||
'ftp_at' => null
|
||||
]);
|
||||
}
|
||||
|
||||
Log::info('Sync log berhasil dimuat/dibuat', ['sync_log_id' => $this->syncLog->id]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Update status saat job dimulai
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
private function updateJobStartStatus(): void
|
||||
{
|
||||
Log::info('Memperbarui status job dimulai');
|
||||
|
||||
$this->syncLog->update([
|
||||
'sync_notes' => $this->syncLog->sync_notes . "\nBatch update seluruh kartu ATM dimulai pada " . now()->format('Y-m-d H:i:s'),
|
||||
'is_sync' => false,
|
||||
'sync_at' => null
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Ambil total jumlah kartu yang akan diproses
|
||||
*
|
||||
* @return int
|
||||
*/
|
||||
private function getTotalCardsCount(): int
|
||||
{
|
||||
Log::info('Menghitung total kartu yang akan diproses', ['filters' => $this->filters]);
|
||||
|
||||
$query = $this->buildCardQuery();
|
||||
$count = $query->count();
|
||||
|
||||
Log::info("Total kartu ditemukan: {$count}");
|
||||
return $count;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build query untuk mengambil kartu berdasarkan filter
|
||||
*
|
||||
* @return \Illuminate\Database\Eloquent\Builder
|
||||
*/
|
||||
private function buildCardQuery()
|
||||
{
|
||||
$query = Atmcard::where('crsts', 1) // Kartu aktif
|
||||
->whereNotNull('accflag')
|
||||
->where('accflag', '!=', '');
|
||||
|
||||
// Terapkan filter default untuk kartu yang perlu update branch/currency
|
||||
if (empty($this->filters) || !isset($this->filters['skip_branch_currency_filter'])) {
|
||||
$query->where(function ($q) {
|
||||
$q->whereNull('branch')
|
||||
->orWhere('branch', '')
|
||||
->orWhereNull('currency')
|
||||
->orWhere('currency', '');
|
||||
});
|
||||
}
|
||||
|
||||
// Terapkan filter tambahan jika ada
|
||||
if (!empty($this->filters)) {
|
||||
foreach ($this->filters as $field => $value) {
|
||||
if ($field === 'skip_branch_currency_filter') {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (is_array($value)) {
|
||||
$query->whereIn($field, $value);
|
||||
} else {
|
||||
$query->where($field, $value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $query;
|
||||
}
|
||||
|
||||
/**
|
||||
* Proses kartu dalam batch
|
||||
*
|
||||
* @param int $totalCards
|
||||
* @return int Jumlah kartu yang berhasil diproses
|
||||
*/
|
||||
private function processCardsInBatches(int $totalCards): int
|
||||
{
|
||||
Log::info('Memulai pemrosesan kartu dalam batch', [
|
||||
'total_cards' => $totalCards,
|
||||
'batch_size' => $this->batchSize
|
||||
]);
|
||||
|
||||
$processedCount = 0;
|
||||
$batchNumber = 1;
|
||||
$totalBatches = ceil($totalCards / $this->batchSize);
|
||||
|
||||
// Proses kartu dalam chunk/batch
|
||||
$this->buildCardQuery()->chunk($this->batchSize, function ($cards) use (&$processedCount, &$batchNumber, $totalBatches, $totalCards) {
|
||||
Log::info("Memproses batch {$batchNumber}/{$totalBatches}", [
|
||||
'cards_in_batch' => $cards->count(),
|
||||
'processed_so_far' => $processedCount
|
||||
]);
|
||||
|
||||
try {
|
||||
// Dispatch job untuk setiap kartu dalam batch dengan delay
|
||||
foreach ($cards as $index => $card) {
|
||||
// Hitung delay berdasarkan nomor batch dan index untuk menyebar eksekusi job
|
||||
$delay = (($batchNumber - 1) * $this->batchSize + $index) % self::MAX_DELAY_SPREAD;
|
||||
$delay += self::DELAY_BETWEEN_JOBS; // Tambah delay minimum
|
||||
|
||||
// Dispatch job UpdateAtmCardBranchCurrencyJob
|
||||
UpdateAtmCardBranchCurrencyJob::dispatch($card, $this->syncLog->id)
|
||||
->delay(now()->addSeconds($delay))
|
||||
->onQueue('default');
|
||||
|
||||
$processedCount++;
|
||||
}
|
||||
|
||||
// Update progress di log setiap 10 batch
|
||||
if ($batchNumber % 10 === 0) {
|
||||
$this->updateProgressStatus($processedCount, $totalCards, $batchNumber, $totalBatches);
|
||||
}
|
||||
|
||||
Log::info("Batch {$batchNumber} berhasil dijadwalkan", [
|
||||
'cards_scheduled' => $cards->count(),
|
||||
'total_processed' => $processedCount
|
||||
]);
|
||||
|
||||
} catch (Exception $e) {
|
||||
Log::error("Error saat memproses batch {$batchNumber}: " . $e->getMessage(), [
|
||||
'batch_number' => $batchNumber,
|
||||
'cards_count' => $cards->count(),
|
||||
'error' => $e->getMessage()
|
||||
]);
|
||||
throw $e;
|
||||
}
|
||||
|
||||
$batchNumber++;
|
||||
});
|
||||
|
||||
Log::info('Selesai memproses semua batch', [
|
||||
'total_processed' => $processedCount,
|
||||
'total_batches' => $batchNumber - 1
|
||||
]);
|
||||
|
||||
return $processedCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update status progress pemrosesan
|
||||
*
|
||||
* @param int $processedCount
|
||||
* @param int $totalCards
|
||||
* @param int $batchNumber
|
||||
* @param int $totalBatches
|
||||
* @return void
|
||||
*/
|
||||
private function updateProgressStatus(int $processedCount, int $totalCards, int $batchNumber, int $totalBatches): void
|
||||
{
|
||||
Log::info('Memperbarui status progress', [
|
||||
'processed' => $processedCount,
|
||||
'total' => $totalCards,
|
||||
'batch' => $batchNumber,
|
||||
'total_batches' => $totalBatches
|
||||
]);
|
||||
|
||||
$percentage = round(($processedCount / $totalCards) * 100, 2);
|
||||
$progressNote = "\nProgress: {$processedCount}/{$totalCards} kartu dijadwalkan ({$percentage}%) - Batch {$batchNumber}/{$totalBatches}";
|
||||
|
||||
$this->syncLog->update([
|
||||
'sync_notes' => $this->syncLog->sync_notes . $progressNote
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Update status saat job selesai
|
||||
*
|
||||
* @param int $totalCards
|
||||
* @param int $processedCount
|
||||
* @return void
|
||||
*/
|
||||
private function updateJobCompletedStatus(int $totalCards, int $processedCount): void
|
||||
{
|
||||
Log::info('Memperbarui status job selesai', [
|
||||
'total_cards' => $totalCards,
|
||||
'processed_count' => $processedCount
|
||||
]);
|
||||
|
||||
$completionNote = "\nBatch update selesai pada " . now()->format('Y-m-d H:i:s') .
|
||||
" - Total {$processedCount} kartu dari {$totalCards} berhasil dijadwalkan untuk update";
|
||||
|
||||
$this->syncLog->update([
|
||||
'is_sync' => true,
|
||||
'sync_at' => now(),
|
||||
'sync_notes' => $this->syncLog->sync_notes . $completionNote
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Update status saat job gagal
|
||||
*
|
||||
* @param string $errorMessage
|
||||
* @return void
|
||||
*/
|
||||
private function updateJobFailedStatus(string $errorMessage): void
|
||||
{
|
||||
Log::error('Memperbarui status job gagal', ['error' => $errorMessage]);
|
||||
|
||||
if ($this->syncLog) {
|
||||
$failureNote = "\nBatch update gagal pada " . now()->format('Y-m-d H:i:s') .
|
||||
" - Error: {$errorMessage}";
|
||||
|
||||
$this->syncLog->update([
|
||||
'is_sync' => false,
|
||||
'sync_notes' => $this->syncLog->sync_notes . $failureNote
|
||||
]);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user