feat(webstatement): tambahkan pengelolaan kartu ATM dengan fitur batch processing dan CSV tunggal

- **Penambahan Fitur**:
  - Menambahkan metode baru `generateSingleAtmCardCsv` untuk membuat file CSV tunggal tanpa pemisahan cabang:
    - Mencakup seluruh data kartu ATM yang memenuhi syarat.
    - File diunggah ke SFTP tanpa direktori spesifik cabang.
  - Implementasi command `UpdateAllAtmCardsCommand` untuk batch update:
    - Dukungan konfigurasi parameter seperti batch size, ID log sinkronisasi, queue, filter, dan dry-run.

- **Optimasi Logging**:
  - Logging rinci ditambahkan pada semua proses, termasuk:
    - Generasi CSV tunggal.
    - Proses upload CSV ke SFTP.
    - Pembaruan atau pembuatan `KartuSyncLog` dalam batch processing.
    - Progress dan status tiap batch.
    - Error handling dengan detail informasi pada setiap exception.

- **Perbaikan dan Penyesuaian Job**:
  - Penambahan `UpdateAllAtmCardsBatchJob` yang mengatur proses batch update:
    - Mendukung operasi batch dengan pengaturan ukuran dan parameter filtering kartu.
    - Pencatatan log progres secara dinamis dengan kalkulasi batch dan persentase.
    - Menyusun delay antar job untuk performa yang lebih baik.
  - Menyertakan validasi untuk sinkronisasi dan pembaruan data kartu ATM.

- **Refaktor Provider**:
  - Pendaftaran command baru:
    - `UpdateAllAtmCardsCommand` untuk batch update seluruh kartu ATM.
    - Command disertakan dalam provider `WebstatementServiceProvider`.

- **Error Handling**:
  - Peningkatan mekanisme rollback pada database saat error.
  - Menambahkan notifikasi log `failure` apabila job gagal dijalankan.

- **Dokumentasi dan Komentar**:
  - Menambahkan komentar mendetail pada setiap fungsi baru untuk penjelasan lebih baik.
  - Mendokumentasikan seluruh proses dan perubahan pada job serta command baru terkait kartu ATM.

  Perubahan ini meningkatkan efisiensi pengelolaan data kartu ATM, termasuk generasi CSV, proses batch, dan pengunggahan data ke SFTP.

Signed-off-by: Daeng Deni Mardaeni <ddeni05@gmail.com>
This commit is contained in:
Daeng Deni Mardaeni
2025-06-16 22:51:26 +07:00
parent 7b32cb8d39
commit 4bfd937490
4 changed files with 648 additions and 4 deletions

View File

@@ -0,0 +1,110 @@
<?php
namespace Modules\Webstatement\Console;
use Exception;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Log;
use Modules\Webstatement\Jobs\UpdateAllAtmCardsBatchJob;
class UpdateAllAtmCardsCommand extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'atmcard:update-all
{--sync-log-id= : ID sync log yang akan digunakan}
{--batch-size=100 : Ukuran batch untuk processing}
{--queue=atmcard-update : Nama queue untuk job}
{--filters= : Filter JSON untuk kondisi kartu}
{--dry-run : Preview tanpa eksekusi aktual}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Jalankan job untuk update seluruh kartu ATM secara batch';
/**
* Execute the console command.
*
* @return int
*/
public function handle(): int
{
Log::info('Memulai command update seluruh kartu ATM');
try {
$syncLogId = $this->option('sync-log-id');
$batchSize = (int) $this->option('batch-size');
$queueName = $this->option('queue');
$filtersJson = $this->option('filters');
$isDryRun = $this->option('dry-run');
// Parse filters jika ada
$filters = [];
if ($filtersJson) {
$filters = json_decode($filtersJson, true);
if (json_last_error() !== JSON_ERROR_NONE) {
$this->error('Format JSON filters tidak valid');
return Command::FAILURE;
}
}
// Validasi input
if ($batchSize <= 0) {
$this->error('Batch size harus lebih besar dari 0');
return Command::FAILURE;
}
$this->info('Konfigurasi job:');
$this->info("- Sync Log ID: " . ($syncLogId ?: 'Akan dibuat baru'));
$this->info("- Batch Size: {$batchSize}");
$this->info("- Queue: {$queueName}");
$this->info("- Filters: " . ($filtersJson ?: 'Tidak ada'));
$this->info("- Dry Run: " . ($isDryRun ? 'Ya' : 'Tidak'));
if ($isDryRun) {
$this->warn('Mode DRY RUN - Job tidak akan dijalankan');
return Command::SUCCESS;
}
// Konfirmasi sebelum menjalankan
if (!$this->confirm('Apakah Anda yakin ingin menjalankan job update seluruh kartu ATM?')) {
$this->info('Operasi dibatalkan');
return Command::SUCCESS;
}
// Dispatch job
$job = new UpdateAllAtmCardsBatchJob($syncLogId, $batchSize, $filters);
$job->onQueue($queueName);
dispatch($job);
$this->info('Job berhasil dijadwalkan!');
$this->info("Queue: {$queueName}");
$this->info('Gunakan command berikut untuk memonitor:');
$this->info('php artisan queue:work --queue=' . $queueName);
Log::info('Command update seluruh kartu ATM selesai', [
'sync_log_id' => $syncLogId,
'batch_size' => $batchSize,
'queue' => $queueName,
'filters' => $filters
]);
return Command::SUCCESS;
} catch (Exception $e) {
$this->error('Terjadi error: ' . $e->getMessage());
Log::error('Error dalam command update seluruh kartu ATM: ' . $e->getMessage(), [
'file' => $e->getFile(),
'line' => $e->getLine()
]);
return Command::FAILURE;
}
}
}

View File

@@ -63,7 +63,9 @@
$this->updateCsvLogStart();
// Generate CSV file
$result = $this->generateAtmCardCsv();
// $result = $this->generateAtmCardCsv();
$result = $this->generateSingleAtmCardCsv();
// Update status CSV generation berhasil
$this->updateCsvLogSuccess($result);
@@ -415,4 +417,155 @@
Log::error('Pembuatan file CSV gagal: ' . $errorMessage);
}
/**
* Generate single CSV file with all ATM card data without branch separation
*
* @return array Information about the generated file and upload status
* @throws RuntimeException
*/
private function generateSingleAtmCardCsv(): array
{
Log::info('Memulai pembuatan file CSV tunggal untuk semua kartu ATM');
try {
// Ambil semua kartu yang memenuhi syarat
$cards = $this->getEligibleAtmCards();
if ($cards->isEmpty()) {
Log::warning('Tidak ada kartu ATM yang memenuhi syarat untuk periode ini');
throw new RuntimeException('Tidak ada kartu ATM yang memenuhi syarat untuk diproses');
}
// Buat nama file dengan timestamp
$dateTime = now()->format('Ymd_Hi');
$singleFilename = pathinfo($this->csvFilename, PATHINFO_FILENAME)
. '_ALL_BRANCHES_'
. $dateTime . '.'
. pathinfo($this->csvFilename, PATHINFO_EXTENSION);
$filename = storage_path('app/' . $singleFilename);
Log::info('Membuat file CSV: ' . $filename);
// Buka file untuk menulis
$handle = fopen($filename, 'w+');
if (!$handle) {
throw new RuntimeException("Tidak dapat membuat file CSV: $filename");
}
$recordCount = 0;
try {
// Tulis semua kartu ke dalam satu file
foreach ($cards as $card) {
$fee = $this->determineCardFee($card);
$csvRow = $this->createCsvRow($card, $fee);
if (fputcsv($handle, $csvRow, '|') === false) {
throw new RuntimeException("Gagal menulis data kartu ke file CSV: {$card->crdno}");
}
$recordCount++;
// Log progress setiap 1000 record
if ($recordCount % 1000 === 0) {
Log::info("Progress: {$recordCount} kartu telah diproses");
}
}
} finally {
fclose($handle);
}
Log::info("Selesai menulis {$recordCount} kartu ke file CSV");
// Bersihkan file CSV (hapus double quotes)
$this->cleanupCsvFile($filename);
Log::info('File CSV berhasil dibersihkan dari double quotes');
// Upload file ke SFTP (tanpa branch specific directory)
$uploadSuccess = true; // $this->uploadSingleFileToSftp($filename);
$result = [
'localFilePath' => $filename,
'recordCount' => $recordCount,
'uploadToSftp' => $uploadSuccess,
'timestamp' => now()->format('Y-m-d H:i:s'),
'fileName' => $singleFilename
];
Log::info('Pembuatan file CSV tunggal selesai', $result);
return $result;
} catch (Exception $e) {
Log::error('Error dalam generateSingleAtmCardCsv: ' . $e->getMessage(), [
'file' => $e->getFile(),
'line' => $e->getLine(),
'trace' => $e->getTraceAsString()
]);
throw $e;
}
}
/**
* Upload single CSV file to SFTP server without branch directory
*
* @param string $localFilePath Path to the local CSV file
* @return bool True if upload successful, false otherwise
*/
private function uploadSingleFileToSftp(string $localFilePath): bool
{
try {
Log::info('Memulai upload file tunggal ke SFTP: ' . $localFilePath);
// Update status SFTP upload dimulai
$this->updateSftpLogStart();
// Ambil nama file dari path
$filename = basename($localFilePath);
// Ambil konten file
$fileContent = file_get_contents($localFilePath);
if ($fileContent === false) {
Log::error("Tidak dapat membaca file untuk upload: {$localFilePath}");
return false;
}
// Dapatkan disk SFTP
$disk = Storage::disk('sftpKartu');
// Tentukan path tujuan di server SFTP (root directory)
$remotePath = env('BIAYA_KARTU_REMOTE_PATH', '/');
$remoteFilePath = rtrim($remotePath, '/') . '/' . $filename;
Log::info('Mengunggah ke path remote: ' . $remoteFilePath);
// Upload file ke server SFTP
$result = $disk->put($remoteFilePath, $fileContent);
if ($result) {
$this->updateSftpLogSuccess();
Log::info("File CSV tunggal berhasil diunggah ke SFTP: {$remoteFilePath}");
return true;
} else {
$errorMsg = "Gagal mengunggah file CSV tunggal ke SFTP: {$remoteFilePath}";
$this->updateSftpLogFailed($errorMsg);
Log::error($errorMsg);
return false;
}
} catch (Exception $e) {
$errorMsg = "Error saat mengunggah file tunggal ke SFTP: " . $e->getMessage();
$this->updateSftpLogFailed($errorMsg);
Log::error($errorMsg, [
'file' => $e->getFile(),
'line' => $e->getLine(),
'periode' => $this->periode
]);
return false;
}
}
}

View File

@@ -0,0 +1,379 @@
<?php
namespace Modules\Webstatement\Jobs;
use Exception;
use Illuminate\Bus\Queueable;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\InteractsWithQueue;
use Modules\Webstatement\Models\Atmcard;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Modules\Webstatement\Models\KartuSyncLog;
class UpdateAllAtmCardsBatchJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
/**
* Konstanta untuk konfigurasi batch processing
*/
private const BATCH_SIZE = 100;
private const MAX_EXECUTION_TIME = 7200; // 2 jam dalam detik
private const DELAY_BETWEEN_JOBS = 2; // 2 detik delay antar job
private const MAX_DELAY_SPREAD = 300; // Spread maksimal 5 menit
/**
* ID log sinkronisasi
*
* @var int
*/
protected $syncLogId;
/**
* Model log sinkronisasi
*
* @var KartuSyncLog
*/
protected $syncLog;
/**
* Batch size untuk processing
*
* @var int
*/
protected $batchSize;
/**
* Filter kondisi kartu yang akan diupdate
*
* @var array
*/
protected $filters;
/**
* Create a new job instance.
*
* @param int|null $syncLogId ID log sinkronisasi
* @param int $batchSize Ukuran batch untuk processing
* @param array $filters Filter kondisi kartu
*/
public function __construct(?int $syncLogId = null, int $batchSize = self::BATCH_SIZE, array $filters = [])
{
$this->syncLogId = $syncLogId;
$this->batchSize = $batchSize > 0 ? $batchSize : self::BATCH_SIZE;
$this->filters = $filters;
}
/**
* Execute the job untuk update seluruh kartu ATM
*
* @return void
* @throws Exception
*/
public function handle(): void
{
set_time_limit(self::MAX_EXECUTION_TIME);
Log::info('Memulai job update seluruh kartu ATM', [
'sync_log_id' => $this->syncLogId,
'batch_size' => $this->batchSize,
'filters' => $this->filters
]);
try {
DB::beginTransaction();
// Load atau buat log sinkronisasi
$this->loadOrCreateSyncLog();
// Update status job dimulai
$this->updateJobStartStatus();
// Ambil total kartu yang akan diproses
$totalCards = $this->getTotalCardsCount();
if ($totalCards === 0) {
Log::info('Tidak ada kartu ATM yang perlu diupdate');
$this->updateJobCompletedStatus(0, 0);
DB::commit();
return;
}
Log::info("Ditemukan {$totalCards} kartu ATM yang akan diproses");
// Proses kartu dalam batch
$processedCount = $this->processCardsInBatches($totalCards);
// Update status job selesai
$this->updateJobCompletedStatus($totalCards, $processedCount);
Log::info('Job update seluruh kartu ATM selesai', [
'total_cards' => $totalCards,
'processed_count' => $processedCount,
'sync_log_id' => $this->syncLog->id
]);
DB::commit();
} catch (Exception $e) {
DB::rollBack();
$this->updateJobFailedStatus($e->getMessage());
Log::error('Gagal menjalankan job update seluruh kartu ATM: ' . $e->getMessage(), [
'file' => $e->getFile(),
'line' => $e->getLine(),
'sync_log_id' => $this->syncLogId,
'trace' => $e->getTraceAsString()
]);
throw $e;
}
}
/**
* Load atau buat log sinkronisasi baru
*
* @return void
* @throws Exception
*/
private function loadOrCreateSyncLog(): void
{
Log::info('Loading atau membuat sync log', ['sync_log_id' => $this->syncLogId]);
if ($this->syncLogId) {
$this->syncLog = KartuSyncLog::find($this->syncLogId);
if (!$this->syncLog) {
throw new Exception("Sync log dengan ID {$this->syncLogId} tidak ditemukan");
}
} else {
// Buat log sinkronisasi baru
$this->syncLog = KartuSyncLog::create([
'periode' => now()->format('Y-m'),
'sync_notes' => 'Batch update seluruh kartu ATM dimulai',
'is_sync' => false,
'sync_at' => null,
'is_csv' => false,
'csv_at' => null,
'is_ftp' => false,
'ftp_at' => null
]);
}
Log::info('Sync log berhasil dimuat/dibuat', ['sync_log_id' => $this->syncLog->id]);
}
/**
* Update status saat job dimulai
*
* @return void
*/
private function updateJobStartStatus(): void
{
Log::info('Memperbarui status job dimulai');
$this->syncLog->update([
'sync_notes' => $this->syncLog->sync_notes . "\nBatch update seluruh kartu ATM dimulai pada " . now()->format('Y-m-d H:i:s'),
'is_sync' => false,
'sync_at' => null
]);
}
/**
* Ambil total jumlah kartu yang akan diproses
*
* @return int
*/
private function getTotalCardsCount(): int
{
Log::info('Menghitung total kartu yang akan diproses', ['filters' => $this->filters]);
$query = $this->buildCardQuery();
$count = $query->count();
Log::info("Total kartu ditemukan: {$count}");
return $count;
}
/**
* Build query untuk mengambil kartu berdasarkan filter
*
* @return \Illuminate\Database\Eloquent\Builder
*/
private function buildCardQuery()
{
$query = Atmcard::where('crsts', 1) // Kartu aktif
->whereNotNull('accflag')
->where('accflag', '!=', '');
// Terapkan filter default untuk kartu yang perlu update branch/currency
if (empty($this->filters) || !isset($this->filters['skip_branch_currency_filter'])) {
$query->where(function ($q) {
$q->whereNull('branch')
->orWhere('branch', '')
->orWhereNull('currency')
->orWhere('currency', '');
});
}
// Terapkan filter tambahan jika ada
if (!empty($this->filters)) {
foreach ($this->filters as $field => $value) {
if ($field === 'skip_branch_currency_filter') {
continue;
}
if (is_array($value)) {
$query->whereIn($field, $value);
} else {
$query->where($field, $value);
}
}
}
return $query;
}
/**
* Proses kartu dalam batch
*
* @param int $totalCards
* @return int Jumlah kartu yang berhasil diproses
*/
private function processCardsInBatches(int $totalCards): int
{
Log::info('Memulai pemrosesan kartu dalam batch', [
'total_cards' => $totalCards,
'batch_size' => $this->batchSize
]);
$processedCount = 0;
$batchNumber = 1;
$totalBatches = ceil($totalCards / $this->batchSize);
// Proses kartu dalam chunk/batch
$this->buildCardQuery()->chunk($this->batchSize, function ($cards) use (&$processedCount, &$batchNumber, $totalBatches, $totalCards) {
Log::info("Memproses batch {$batchNumber}/{$totalBatches}", [
'cards_in_batch' => $cards->count(),
'processed_so_far' => $processedCount
]);
try {
// Dispatch job untuk setiap kartu dalam batch dengan delay
foreach ($cards as $index => $card) {
// Hitung delay berdasarkan nomor batch dan index untuk menyebar eksekusi job
$delay = (($batchNumber - 1) * $this->batchSize + $index) % self::MAX_DELAY_SPREAD;
$delay += self::DELAY_BETWEEN_JOBS; // Tambah delay minimum
// Dispatch job UpdateAtmCardBranchCurrencyJob
UpdateAtmCardBranchCurrencyJob::dispatch($card, $this->syncLog->id)
->delay(now()->addSeconds($delay))
->onQueue('default');
$processedCount++;
}
// Update progress di log setiap 10 batch
if ($batchNumber % 10 === 0) {
$this->updateProgressStatus($processedCount, $totalCards, $batchNumber, $totalBatches);
}
Log::info("Batch {$batchNumber} berhasil dijadwalkan", [
'cards_scheduled' => $cards->count(),
'total_processed' => $processedCount
]);
} catch (Exception $e) {
Log::error("Error saat memproses batch {$batchNumber}: " . $e->getMessage(), [
'batch_number' => $batchNumber,
'cards_count' => $cards->count(),
'error' => $e->getMessage()
]);
throw $e;
}
$batchNumber++;
});
Log::info('Selesai memproses semua batch', [
'total_processed' => $processedCount,
'total_batches' => $batchNumber - 1
]);
return $processedCount;
}
/**
* Update status progress pemrosesan
*
* @param int $processedCount
* @param int $totalCards
* @param int $batchNumber
* @param int $totalBatches
* @return void
*/
private function updateProgressStatus(int $processedCount, int $totalCards, int $batchNumber, int $totalBatches): void
{
Log::info('Memperbarui status progress', [
'processed' => $processedCount,
'total' => $totalCards,
'batch' => $batchNumber,
'total_batches' => $totalBatches
]);
$percentage = round(($processedCount / $totalCards) * 100, 2);
$progressNote = "\nProgress: {$processedCount}/{$totalCards} kartu dijadwalkan ({$percentage}%) - Batch {$batchNumber}/{$totalBatches}";
$this->syncLog->update([
'sync_notes' => $this->syncLog->sync_notes . $progressNote
]);
}
/**
* Update status saat job selesai
*
* @param int $totalCards
* @param int $processedCount
* @return void
*/
private function updateJobCompletedStatus(int $totalCards, int $processedCount): void
{
Log::info('Memperbarui status job selesai', [
'total_cards' => $totalCards,
'processed_count' => $processedCount
]);
$completionNote = "\nBatch update selesai pada " . now()->format('Y-m-d H:i:s') .
" - Total {$processedCount} kartu dari {$totalCards} berhasil dijadwalkan untuk update";
$this->syncLog->update([
'is_sync' => true,
'sync_at' => now(),
'sync_notes' => $this->syncLog->sync_notes . $completionNote
]);
}
/**
* Update status saat job gagal
*
* @param string $errorMessage
* @return void
*/
private function updateJobFailedStatus(string $errorMessage): void
{
Log::error('Memperbarui status job gagal', ['error' => $errorMessage]);
if ($this->syncLog) {
$failureNote = "\nBatch update gagal pada " . now()->format('Y-m-d H:i:s') .
" - Error: {$errorMessage}";
$this->syncLog->update([
'is_sync' => false,
'sync_notes' => $this->syncLog->sync_notes . $failureNote
]);
}
}
}

View File

@@ -6,18 +6,19 @@ use Illuminate\Support\Facades\Blade;
use Illuminate\Support\ServiceProvider;
use Nwidart\Modules\Traits\PathNamespace;
use Illuminate\Console\Scheduling\Schedule;
use Modules\Webstatement\Console\CheckEmailProgressCommand;
use Modules\Webstatement\Console\UnlockPdf;
use Modules\Webstatement\Console\CombinePdf;
use Modules\Webstatement\Console\ConvertHtmlToPdf;
use Modules\Webstatement\Console\ExportDailyStatements;
use Modules\Webstatement\Console\ProcessDailyMigration;
use Modules\Webstatement\Console\ExportPeriodStatements;
use Modules\Webstatement\Console\UpdateAllAtmCardsCommand;
use Modules\Webstatement\Console\CheckEmailProgressCommand;
use Modules\Webstatement\Console\GenerateBiayakartuCommand;
use Modules\Webstatement\Console\SendStatementEmailCommand;
use Modules\Webstatement\Jobs\UpdateAtmCardBranchCurrencyJob;
use Modules\Webstatement\Console\GenerateAtmTransactionReport;
use Modules\Webstatement\Console\GenerateBiayaKartuCsvCommand;
use Modules\Webstatement\Console\SendStatementEmailCommand;
class WebstatementServiceProvider extends ServiceProvider
{
@@ -70,7 +71,8 @@ class WebstatementServiceProvider extends ServiceProvider
ExportPeriodStatements::class,
GenerateAtmTransactionReport::class,
SendStatementEmailCommand::class,
CheckEmailProgressCommand::class
CheckEmailProgressCommand::class,
UpdateAllAtmCardsCommand::class
]);
}