feat(webstatement): optimalkan proses migrasi data SFTP
- Menambahkan parameter `$periods` untuk mendukung proses dinamis berdasarkan folder periode dalam SFTP.
- Memperkenalkan penggunaan `Storage::disk('sftpStatement')` untuk membaca file dari SFTP.
- Menyortir folder periode secara menurun berdasarkan tanggal.
- Memproses tiap entitas data (Customer, Account, Arrangement, Bill Details, dll.) berdasarkan file dalam folder periode.
- Menambah log untuk mencatat jumlah proses yang berhasil, baris error, dan peringatan tentang ketidaksesuaian jumlah kolom.
- Menyisipkan mekanisme untuk mengabaikan folder `_parameter`.
- Mengubah pendekatan dari membaca file lokal menjadi menggunakan file sementara (temp).
- Memperbaiki bug pada metode yang sebelumnya menggunakan jalur file statis.
- Memastikan penanganan file CSV yang lebih fleksibel, termasuk kasus di mana jumlah kolom dalam baris melebihi ekspektasi.
- Menambah error handling untuk mencatat baris-baris bermasalah dan menghindari interupsi proses.
Signed-off-by: Daeng Deni Mardaeni <ddeni05@gmail.com>
This commit is contained in:
Daeng Deni Mardaeni
@@ -4,6 +4,7 @@ namespace Modules\Webstatement\Http\Controllers;
|
||||
|
||||
use App\Http\Controllers\Controller;
|
||||
use Exception;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use Log;
|
||||
use Modules\Webstatement\Jobs\ProcessAccountDataJob;
|
||||
use Modules\Webstatement\Jobs\ProcessArrangementDataJob;
|
||||
@@ -18,49 +19,54 @@ use Modules\Webstatement\Jobs\ProcessTransactionDataJob;
|
||||
class MigrasiController extends Controller
|
||||
{
|
||||
|
||||
public function processArrangementData()
|
||||
public function processArrangementData($periods)
|
||||
{
|
||||
try {
|
||||
ProcessArrangementDataJob::dispatch();
|
||||
ProcessArrangementDataJob::dispatch($periods);
|
||||
return response()->json(['message' => 'Data Arrangement processing job has been successfully']);
|
||||
} catch (Exception $e) {
|
||||
return response()->json(['error' => $e->getMessage()], 500);
|
||||
}
|
||||
}
|
||||
|
||||
public function processCustomerData()
|
||||
public function processCustomerData($periods)
|
||||
{
|
||||
$filePath = storage_path('app/20240901.ST.CUSTOMER.csv'); // Adjust this path as needed
|
||||
try {
|
||||
ProcessCustomerDataJob::dispatch();
|
||||
return response()->json(['message' => 'Data Customer processing job has been successfully']);
|
||||
// Pass the periods to the job for processing
|
||||
ProcessCustomerDataJob::dispatch($periods);
|
||||
|
||||
return response()->json([
|
||||
'message' => 'Data Customer processing job has been successfully queued',
|
||||
'periods' => $periods
|
||||
]);
|
||||
} catch (Exception $e) {
|
||||
Log::error('Error in processCustomerData: ' . $e->getMessage());
|
||||
return response()->json(['error' => $e->getMessage()], 500);
|
||||
}
|
||||
}
|
||||
|
||||
public function processBillDetailData()
|
||||
public function processBillDetailData($periods)
|
||||
{
|
||||
try {
|
||||
ProcessBillDetailDataJob::dispatch();
|
||||
ProcessBillDetailDataJob::dispatch($periods);
|
||||
return response()->json(['message' => 'Data Bill Details processing job has been successfully']);
|
||||
} catch (Exception $e) {
|
||||
return response()->json(['error' => $e->getMessage()], 500);
|
||||
}
|
||||
}
|
||||
|
||||
public function processAccountData(){
|
||||
public function processAccountData($periods){
|
||||
try{
|
||||
ProcessAccountDataJob::dispatch();
|
||||
ProcessAccountDataJob::dispatch($periods);
|
||||
return response()->json(['message' => 'Data Account processing job has been successfully']);
|
||||
} catch (Exception $e) {
|
||||
return response()->json(['error' => $e->getMessage()], 500);
|
||||
}
|
||||
}
|
||||
|
||||
public function processTransactionData(){
|
||||
public function processTransactionData($periods){
|
||||
try{
|
||||
ProcessTransactionDataJob::dispatch();
|
||||
ProcessTransactionDataJob::dispatch($periods);
|
||||
Log::info('Data Transaction processing job has been successfully');
|
||||
return response()->json(['message' => 'Data Transaction processing job has been successfully']);
|
||||
} catch (Exception $e) {
|
||||
@@ -68,37 +74,37 @@ class MigrasiController extends Controller
|
||||
}
|
||||
}
|
||||
|
||||
public function processFundsTransferData(){
|
||||
public function processFundsTransferData($periods){
|
||||
try{
|
||||
ProcessFundsTransferDataJob::dispatch();
|
||||
ProcessFundsTransferDataJob::dispatch($periods);
|
||||
return response()->json(['message' => 'Data Funds Transfer processing job has been successfully']);
|
||||
} catch (Exception $e) {
|
||||
return response()->json(['error' => $e->getMessage()], 500);
|
||||
}
|
||||
}
|
||||
|
||||
public function processStmtNarrParamData()
|
||||
public function processStmtNarrParamData($periods)
|
||||
{
|
||||
try {
|
||||
ProcessStmtNarrParamDataJob::dispatch();
|
||||
ProcessStmtNarrParamDataJob::dispatch($periods);
|
||||
return response()->json(['message' => 'Data TempStmtNarrParam processing job has been successfully']);
|
||||
} catch (Exception $e) {
|
||||
return response()->json(['error' => $e->getMessage()], 500);
|
||||
}
|
||||
}
|
||||
|
||||
public function processStmtNarrFormatData(){
|
||||
public function processStmtNarrFormatData($periods){
|
||||
try {
|
||||
ProcessStmtNarrFormatDataJob::dispatch();
|
||||
ProcessStmtNarrFormatDataJob::dispatch($periods);
|
||||
return response()->json(['message' => 'Data TempStmtNarrFormat processing job has been successfully']);
|
||||
} catch (Exception $e) {
|
||||
return response()->json(['error' => $e->getMessage()], 500);
|
||||
}
|
||||
}
|
||||
|
||||
public function processStmtEntryData(){
|
||||
public function processStmtEntryData($periods){
|
||||
try {
|
||||
ProcessStmtEntryDataJob::dispatch();
|
||||
ProcessStmtEntryDataJob::dispatch($periods);
|
||||
return response()->json(['message' => 'Data TempStmtEntry processing job has been successfully']);
|
||||
} catch (Exception $e) {
|
||||
return response()->json(['error' => $e->getMessage()], 500);
|
||||
@@ -108,15 +114,34 @@ class MigrasiController extends Controller
|
||||
|
||||
public function index()
|
||||
{
|
||||
$this->processCustomerData();
|
||||
$this->processAccountData();
|
||||
$this->processArrangementData();
|
||||
$this->processBillDetailData();
|
||||
$this->processTransactionData();
|
||||
$this->processFundsTransferData();
|
||||
$this->processStmtNarrParamData();
|
||||
$this->processStmtNarrFormatData();
|
||||
$this->processStmtEntryData();
|
||||
$disk = Storage::disk('sftpStatement');
|
||||
|
||||
// Get all directories (periods) in the SFTP disk
|
||||
$allDirectories = $disk->directories();
|
||||
|
||||
// Filter out the _parameter folder
|
||||
$periods = array_filter($allDirectories, function($dir) {
|
||||
return $dir !== '_parameter';
|
||||
});
|
||||
|
||||
// Sort periods by date (descending)
|
||||
usort($periods, function($a, $b) {
|
||||
return strcmp($b, $a); // Reverse comparison for descending order
|
||||
});
|
||||
|
||||
if (empty($periods)) {
|
||||
return response()->json(['message' => 'No valid period folders found in SFTP storage'], 404);
|
||||
}
|
||||
|
||||
$this->processCustomerData($periods);
|
||||
$this->processAccountData($periods);
|
||||
$this->processArrangementData($periods);
|
||||
$this->processBillDetailData($periods);
|
||||
$this->processTransactionData($periods);
|
||||
$this->processFundsTransferData($periods);
|
||||
$this->processStmtNarrParamData($periods);
|
||||
$this->processStmtNarrFormatData($periods);
|
||||
$this->processStmtEntryData($periods);
|
||||
|
||||
return response()->json(['message' => 'Data processing job has been successfully']);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user