daengdeni/webstatement
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(jobs): simplify jobs and controllers by replacing period array with single period parameter

Browse Source 
- Mengganti parameter `$periods` (array) menjadi `$period` (string) pada semua Job terkait: `ProcessCustomerDataJob`, `ProcessFundsTransferDataJob, etc`.
- Menyederhanakan operasi loop dalam proses data dengan hanya memproses satu periode per eksekusi Job.
- Memodifikasi fungsi controller di `MigrasiController` agar sesuai dengan perubahan parameter dari array ke string.
- Menambahkan pengamanan jika `$period` kosong atau bernilai '_parameter' untuk mencegah proses yang tidak diperlukan.
- Mengurangi duplikasi kode dengan mengeliminasi metode yang mengelola array periode dan menggantinya dengan pendekatan tunggal.

Signed-off-by: Daeng Deni Mardaeni <ddeni05@gmail.com>
This commit is contained in:
Daeng Deni Mardaeni
2025-05-24 19:40:40 +07:00
parent 85b8bfa07b
commit cd447eb019
9 changed files with 761 additions and 799 deletions
Download Patch File Download Diff File Expand all files Collapse all files

540
app/Http/Controllers/MigrasiController.php
View File

@@ -1,272 +1,282 @@
<?php <?php
namespace Modules\Webstatement\Http\Controllers; namespace Modules\Webstatement\Http\Controllers;
use App\Http\Controllers\Controller; use App\Http\Controllers\Controller;
use Exception; use Exception;
use Illuminate\Support\Facades\Storage; use Illuminate\Support\Facades\Storage;
use Log; use Log;
use Modules\Webstatement\Jobs\ProcessAccountDataJob; use Modules\Webstatement\Jobs\ProcessAccountDataJob;
use Modules\Webstatement\Jobs\ProcessArrangementDataJob; use Modules\Webstatement\Jobs\ProcessArrangementDataJob;
use Modules\Webstatement\Jobs\ProcessAtmTransactionJob; use Modules\Webstatement\Jobs\ProcessAtmTransactionJob;
use Modules\Webstatement\Jobs\ProcessBillDetailDataJob; use Modules\Webstatement\Jobs\ProcessBillDetailDataJob;
use Modules\Webstatement\Jobs\ProcessCategoryDataJob; use Modules\Webstatement\Jobs\ProcessCategoryDataJob;
use Modules\Webstatement\Jobs\ProcessCompanyDataJob; use Modules\Webstatement\Jobs\ProcessCompanyDataJob;
use Modules\Webstatement\Jobs\ProcessCustomerDataJob; use Modules\Webstatement\Jobs\ProcessCustomerDataJob;
use Modules\Webstatement\Jobs\ProcessDataCaptureDataJob; use Modules\Webstatement\Jobs\ProcessDataCaptureDataJob;
use Modules\Webstatement\Jobs\ProcessFtTxnTypeConditionJob; use Modules\Webstatement\Jobs\ProcessFtTxnTypeConditionJob;
use Modules\Webstatement\Jobs\ProcessFundsTransferDataJob; use Modules\Webstatement\Jobs\ProcessFundsTransferDataJob;
use Modules\Webstatement\Jobs\ProcessStmtEntryDataJob; use Modules\Webstatement\Jobs\ProcessStmtEntryDataJob;
use Modules\Webstatement\Jobs\ProcessStmtNarrFormatDataJob; use Modules\Webstatement\Jobs\ProcessStmtNarrFormatDataJob;
use Modules\Webstatement\Jobs\ProcessStmtNarrParamDataJob; use Modules\Webstatement\Jobs\ProcessStmtNarrParamDataJob;
use Modules\Webstatement\Jobs\ProcessTellerDataJob; use Modules\Webstatement\Jobs\ProcessTellerDataJob;
use Modules\Webstatement\Jobs\ProcessTransactionDataJob; use Modules\Webstatement\Jobs\ProcessTransactionDataJob;
class MigrasiController extends Controller class MigrasiController extends Controller
{
public function processArrangementData($periods)
{ {
try {
ProcessArrangementDataJob::dispatch($periods); public function processTransactionData($period)
return response()->json(['message' => 'Data Arrangement processing job has been successfully']); {
} catch (Exception $e) { try {
return response()->json(['error' => $e->getMessage()], 500); ProcessTransactionDataJob::dispatch($period);
Log::info('Data Transaction processing job has been successfully');
return response()->json(['message' => 'Data Transaction processing job has been successfully']);
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function processStmtNarrParamData($period)
{
try {
ProcessStmtNarrParamDataJob::dispatch($period);
return response()->json(['message' => 'Data TempStmtNarrParam processing job has been successfully']);
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function processStmtNarrFormatData($period)
{
try {
ProcessStmtNarrFormatDataJob::dispatch($period);
return response()->json(['message' => 'Data TempStmtNarrFormat processing job has been successfully']);
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function ProcessFtTxnTypeConditioData($period)
{
try {
ProcessFtTxnTypeConditionJob::dispatch($period);
return response()->json(['message' => 'FtTxnTypeCondition processing job has been successfully']);
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function index_manual()
{
//$disk = Storage::disk('sftpStatement');
// Get all directories (periods) in the SFTP disk
//$allDirectories = $disk->directories();
//$this->processTransactionData('_parameter');
//$this->processStmtNarrParamData('_parameter');
//$this->processStmtNarrFormatData('_parameter');
//$this->ProcessFtTxnTypeConditioData('_parameter');
// Filter out the _parameter folder
/*$periods = array_filter($allDirectories, function($dir) {
return $dir !== '_parameter';
});*/
$periods = [
/*'20250510',
'20250512',
'20250513',
'20250514',
'20250515',
'20250516',
'20250517',
'20250518',
'20250519',*/
'20250520',
'20250521',
'20250522'
];
// Sort periods by date (descending)
usort($periods, function ($a, $b) {
return strcmp($b, $a); // Reverse comparison for descending order
});
if (empty($periods)) {
return response()->json(['message' => 'No valid period folders found in SFTP storage'], 404);
}
foreach ($periods as $period) {
$this->ProcessCategoryData($period);
$this->ProcessCompanyData($period);
$this->processCustomerData($period);
$this->processAccountData($period);
$this->processStmtEntryData($period);
$this->ProcessDataCaptureData($period);
$this->processFundsTransferData($period);
$this->ProcessTellerData($period);
$this->ProcessAtmTransaction($period);
$this->processArrangementData($period);
$this->processBillDetailData($period);
}
return response()->json(['message' => 'Data processing job has been successfully']);
}
public function ProcessCategoryData($period)
{
try {
ProcessCategoryDataJob::dispatch($period);
return response()->json(['message' => 'Category processing job has been successfully']);
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function ProcessCompanyData($period)
{
try {
ProcessCompanyDataJob::dispatch($period);
return response()->json(['message' => 'Company processing job has been successfully']);
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function processCustomerData($period)
{
try {
// Pass the period to the job for processing
ProcessCustomerDataJob::dispatch($period);
return response()->json([
'message' => 'Data Customer processing job has been successfully queued',
'period' => $period
]);
} catch (Exception $e) {
Log::error('Error in processCustomerData: ' . $e->getMessage());
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function processAccountData($period)
{
try {
ProcessAccountDataJob::dispatch($period);
return response()->json(['message' => 'Data Account processing job has been successfully']);
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function processStmtEntryData($period)
{
try {
ProcessStmtEntryDataJob::dispatch($period);
return response()->json(['message' => 'Stmt Entry processing job has been successfully']);
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function ProcessDataCaptureData($period)
{
try {
ProcessDataCaptureDataJob::dispatch($period);
return response()->json(['message' => 'Data Capture processing job has been successfully']);
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function processFundsTransferData($period)
{
try {
ProcessFundsTransferDataJob::dispatch($period);
return response()->json(['message' => 'Data Funds Transfer processing job has been successfully']);
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function ProcessTellerData($period)
{
try {
ProcessTellerDataJob::dispatch($period);
return response()->json(['message' => 'Teller processing job has been successfully']);
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function ProcessAtmTransaction($period)
{
try {
ProcessAtmTransactionJob::dispatch($period);
return response()->json(['message' => 'AtmTransaction processing job has been successfully']);
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function processArrangementData($period)
{
try {
ProcessArrangementDataJob::dispatch($period);
return response()->json(['message' => 'Data Arrangement processing job has been successfully']);
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function processBillDetailData($period)
{
try {
ProcessBillDetailDataJob::dispatch($period);
return response()->json(['message' => 'Data Bill Details processing job has been successfully']);
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
}
}
/**
* Process data migration for the previous day's period.
*
* This method automatically determines the period based on yesterday's date,
* checks if the corresponding folder exists in SFTP storage, and then
* processes all required data types in the correct sequence.
*
* @return \Illuminate\Http\JsonResponse A JSON response indicating success or failure
* @throws \Exception If the period folder doesn't exist or if any processing job fails
*/
public function index()
{
$disk = Storage::disk('sftpStatement');
$period = date('Ymd', strtotime('-1 day'));
if (!$disk->exists($period)) {
return response()->json(["message" => "Period {$period} folder not found in SFTP storage"], 404);
}
$this->ProcessCategoryData($period);
$this->ProcessCompanyData($period);
$this->processCustomerData($period);
$this->processAccountData($period);
$this->processStmtEntryData($period);
$this->ProcessDataCaptureData($period);
$this->processFundsTransferData($period);
$this->ProcessTellerData($period);
$this->ProcessAtmTransaction($period);
$this->processArrangementData($period);
$this->processBillDetailData($period);
return response()->json(['message' => "Data processing {$period} job has been successfully"]);
} }
} }
public function processCustomerData($periods)
{
try {
// Pass the periods to the job for processing
ProcessCustomerDataJob::dispatch($periods);
return response()->json([
'message' => 'Data Customer processing job has been successfully queued',
'periods' => $periods
]);
} catch (Exception $e) {
Log::error('Error in processCustomerData: ' . $e->getMessage());
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function processBillDetailData($periods)
{
try {
ProcessBillDetailDataJob::dispatch($periods);
return response()->json(['message' => 'Data Bill Details processing job has been successfully']);
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function processAccountData($periods){
try{
ProcessAccountDataJob::dispatch($periods);
return response()->json(['message' => 'Data Account processing job has been successfully']);
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function processTransactionData($periods){
try{
ProcessTransactionDataJob::dispatch($periods);
Log::info('Data Transaction processing job has been successfully');
return response()->json(['message' => 'Data Transaction processing job has been successfully']);
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function processFundsTransferData($periods){
try{
ProcessFundsTransferDataJob::dispatch($periods);
return response()->json(['message' => 'Data Funds Transfer processing job has been successfully']);
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function processStmtNarrParamData($periods)
{
try {
ProcessStmtNarrParamDataJob::dispatch($periods);
return response()->json(['message' => 'Data TempStmtNarrParam processing job has been successfully']);
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function processStmtNarrFormatData($periods){
try {
ProcessStmtNarrFormatDataJob::dispatch($periods);
return response()->json(['message' => 'Data TempStmtNarrFormat processing job has been successfully']);
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function ProcessFtTxnTypeConditioData($periods){
try {
ProcessFtTxnTypeConditionJob::dispatch($periods);
return response()->json(['message' => 'FtTxnTypeCondition processing job has been successfully']);
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function processStmtEntryData($periods){
try {
ProcessStmtEntryDataJob::dispatch($periods);
return response()->json(['message' => 'Stmt Entry processing job has been successfully']);
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function ProcessCompanyData($periods){
try {
ProcessCompanyDataJob::dispatch($periods);
return response()->json(['message' => 'Company processing job has been successfully']);
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function ProcessDataCaptureData($periods){
try {
ProcessDataCaptureDataJob::dispatch($periods);
return response()->json(['message' => 'Data Capture processing job has been successfully']);
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function ProcessCategoryData($periods){
try {
ProcessCategoryDataJob::dispatch($periods);
return response()->json(['message' => 'Category processing job has been successfully']);
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function ProcessTellerData($periods){
try {
ProcessTellerDataJob::dispatch($periods);
return response()->json(['message' => 'Teller processing job has been successfully']);
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function ProcessAtmTransaction($periods){
try {
ProcessAtmTransactionJob::dispatch($periods);
return response()->json(['message' => 'AtmTransaction processing job has been successfully']);
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function index_manual()
{
//$disk = Storage::disk('sftpStatement');
// Get all directories (periods) in the SFTP disk
//$allDirectories = $disk->directories();
//$this->processTransactionData(['_parameter']);
//$this->processStmtNarrParamData(['_parameter']);
//$this->processStmtNarrFormatData(['_parameter']);
//$this->ProcessFtTxnTypeConditioData(['_parameter']);
// Filter out the _parameter folder
/*$periods = array_filter($allDirectories, function($dir) {
return $dir !== '_parameter';
});*/
$periods = [
/*'20250510',
'20250512',
'20250513',
'20250514',
'20250515',
'20250516',
'20250517',
'20250518',
'20250519',*/
'20250520',
'20250521',
'20250522'
];
// Sort periods by date (descending)
usort($periods, function($a, $b) {
return strcmp($b, $a); // Reverse comparison for descending order
});
if (empty($periods)) {
return response()->json(['message' => 'No valid period folders found in SFTP storage'], 404);
}
foreach($periods as $period){
$this->ProcessCategoryData([$period]);
$this->ProcessCompanyData([$period]);
$this->processCustomerData([$period]);
$this->processAccountData([$period]);
$this->processStmtEntryData([$period]);
$this->ProcessDataCaptureData([$period]);
$this->processFundsTransferData([$period]);
$this->ProcessTellerData([$period]);
$this->ProcessAtmTransaction([$period]);
$this->processArrangementData([$period]);
$this->processBillDetailData([$period]);
}
return response()->json(['message' => 'Data processing job has been successfully']);
}
/**
* Process data migration for the previous day's period.
*
* This method automatically determines the period based on yesterday's date,
* checks if the corresponding folder exists in SFTP storage, and then
* processes all required data types in the correct sequence.
*
* @return \Illuminate\Http\JsonResponse A JSON response indicating success or failure
* @throws \Exception If the period folder doesn't exist or if any processing job fails
*/
public function index(){
$disk = Storage::disk('sftpStatement');
$period = date('Ymd', strtotime('-1 day'));
if (!$disk->exists($period)) {
return response()->json(["message" => "Period {$period} folder not found in SFTP storage"], 404);
}
$this->ProcessCategoryData($period);
$this->ProcessCompanyData($period);
$this->processCustomerData([$period]);
$this->processAccountData([$period]);
$this->processStmtEntryData([$period]);
$this->ProcessDataCaptureData([$period]);
$this->processFundsTransferData([$period]);
$this->ProcessTellerData([$period]);
$this->ProcessAtmTransaction([$period]);
$this->processArrangementData([$period]);
$this->processBillDetailData([$period]);
return response()->json(['message' => "Data processing {$period} job has been successfully"]);
}
}

161
app/Jobs/ProcessAccountDataJob.php
View File

@@ -17,117 +17,116 @@
{ {
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $periods; protected $period;
/** /**
* Create a new job instance. * Create a new job instance.
*/ */
public function __construct(array $periods = []) public function __construct(string $period = '')
{ {
$this->periods = $periods; $this->period = $period;
} }
/** /**
* Execute the job. * Execute the job.
*/ */
public function handle(): void public function handle()
: void
{ {
try { try {
set_time_limit(24 * 60 * 60); set_time_limit(24 * 60 * 60);
$disk = Storage::disk('sftpStatement'); $disk = Storage::disk('sftpStatement');
$processedCount = 0; $processedCount = 0;
$errorCount = 0; $errorCount = 0;
if (empty($this->periods)) { if (empty($this->period)) {
Log::warning('No periods provided for account data processing'); Log::warning('No period provided for account data processing');
return; return;
} }
foreach ($this->periods as $period) { // Skip the _parameter folder
// Skip the _parameter folder if ($this->period === '_parameter') {
if ($period === '_parameter') { Log::info("Skipping _parameter folder");
Log::info("Skipping _parameter folder"); return;
continue; }
}
// Construct the filename based on the period folder name // Construct the filename based on the period folder name
$filename = "$period.ST.ACCOUNT.csv"; $filename = "{$this->period}.ST.ACCOUNT.csv";
$filePath = "$period/$filename"; $filePath = "{$this->period}/$filename";
Log::info("Processing account file: $filePath"); Log::info("Processing account file: $filePath");
if (!$disk->exists($filePath)) { if (!$disk->exists($filePath)) {
Log::warning("File not found: $filePath"); Log::warning("File not found: $filePath");
continue; return;
} }
// Create a temporary local copy of the file // Create a temporary local copy of the file
$tempFilePath = storage_path("app/temp_$filename"); $tempFilePath = storage_path("app/temp_$filename");
file_put_contents($tempFilePath, $disk->get($filePath)); file_put_contents($tempFilePath, $disk->get($filePath));
$handle = fopen($tempFilePath, "r"); $handle = fopen($tempFilePath, "r");
if ($handle !== false) { if ($handle !== false) {
$headers = (new Account())->getFillable(); $headers = (new Account())->getFillable();
Log::info('Headers: ' . implode(", ", $headers)); Log::info('Headers: ' . implode(", ", $headers));
$rowCount = 0; $rowCount = 0;
while (($row = fgetcsv($handle, 0, "~")) !== false) { while (($row = fgetcsv($handle, 0, "~")) !== false) {
$rowCount++; $rowCount++;
if (count($headers) === count($row)) { if (count($headers) === count($row)) {
$data = array_combine($headers, $row); $data = array_combine($headers, $row);
// Check if start_year_bal is empty and set it to 0 if so // Check if start_year_bal is empty and set it to 0 if so
if (empty($data['start_year_bal']) || $data['start_year_bal'] == "" || $data['start_year_bal'] == null) { if (empty($data['start_year_bal']) || $data['start_year_bal'] == "" || $data['start_year_bal'] == null) {
$data['start_year_bal'] = 0; $data['start_year_bal'] = 0;
}
if (empty($data['closure_date']) || $data['closure_date'] == "" || $data['closure_date'] == null) {
$data['closure_date'] = null;
}
try {
if ($data['account_number'] !== 'account_number') {
// Use firstOrNew instead of updateOrCreate
$account = Account::firstOrNew(['account_number' => $data['account_number']]);
$account->fill($data);
$account->save();
// Store the opening balances in the AccountBalance model for this period
if (isset($data['open_actual_bal']) || isset($data['open_cleared_bal'])) {
$accountBalance = AccountBalance::firstOrNew([
'account_number' => $data['account_number'],
'period' => $period
]);
// Set the balances
$accountBalance->actual_balance = $data['open_actual_bal'] ?? 0;
$accountBalance->cleared_balance = $data['open_cleared_bal'] ?? 0;
$accountBalance->save();
Log::info("Saved balance for account {$data['account_number']} for period $period");
}
$processedCount++;
}
} catch (Exception $e) {
$errorCount++;
Log::error("Error processing Account at row $rowCount in $filePath: " . $e->getMessage());
}
} else {
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row));
} }
if (empty($data['closure_date']) || $data['closure_date'] == "" || $data['closure_date'] == null) {
$data['closure_date'] = null;
}
try {
if ($data['account_number'] !== 'account_number') {
// Use firstOrNew instead of updateOrCreate
$account = Account::firstOrNew(['account_number' => $data['account_number']]);
$account->fill($data);
$account->save();
// Store the opening balances in the AccountBalance model for this period
if (isset($data['open_actual_bal']) || isset($data['open_cleared_bal'])) {
$accountBalance = AccountBalance::firstOrNew([
'account_number' => $data['account_number'],
'period' => $this->period
]);
// Set the balances
$accountBalance->actual_balance = $data['open_actual_bal'] ?? 0;
$accountBalance->cleared_balance = $data['open_cleared_bal'] ?? 0;
$accountBalance->save();
Log::info("Saved balance for account {$data['account_number']} for period {$this->period}");
}
$processedCount++;
}
} catch (Exception $e) {
$errorCount++;
Log::error("Error processing Account at row $rowCount in $filePath: " . $e->getMessage());
}
} else {
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row));
} }
fclose($handle);
Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors.");
// Clean up the temporary file
unlink($tempFilePath);
} else {
Log::error("Unable to open file: $filePath");
} }
fclose($handle);
Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors.");
// Clean up the temporary file
unlink($tempFilePath);
} else {
Log::error("Unable to open file: $filePath");
} }
Log::info("Account data processing completed. Total processed: $processedCount, Total errors: $errorCount"); Log::info("Account data processing completed. Total processed: $processedCount, Total errors: $errorCount");

73
app/Jobs/ProcessAtmTransactionJob.php
View File

@@ -19,10 +19,10 @@
private const PARAMETER_FOLDER = '_parameter'; private const PARAMETER_FOLDER = '_parameter';
// Konstanta untuk nilai-nilai statis // Konstanta untuk nilai-nilai statis
private const FILE_EXTENSION = '.ST.ATM.TRANSACTION.csv'; private const FILE_EXTENSION = '.ST.ATM.TRANSACTION.csv';
private const CSV_DELIMITER = '~'; private const CSV_DELIMITER = '~';
private const DISK_NAME = 'sftpStatement'; private const DISK_NAME = 'sftpStatement';
private const HEADER_MAP = [ private const HEADER_MAP = [
'id' => 'transaction_id', 'id' => 'transaction_id',
'card_acc_id' => 'card_acc_id', 'card_acc_id' => 'card_acc_id',
'pan_number' => 'pan_number', 'pan_number' => 'pan_number',
@@ -43,30 +43,37 @@
]; ];
// Pemetaan bidang header ke kolom model // Pemetaan bidang header ke kolom model
protected array $periods; protected string $period;
/** /**
* Create a new job instance. * Create a new job instance.
*/ */
public function __construct(array $periods = []) public function __construct(string $period = '')
{ {
$this->periods = $periods; $this->period = $period;
} }
/** /**
* Execute the job. * Execute the job.
*/ */
public function handle(): void public function handle()
: void
{ {
try { try {
set_time_limit(24 * 60 * 60); set_time_limit(24 * 60 * 60);
if (empty($this->periods)) { if (empty($this->period)) {
Log::warning('No periods provided for ATM transaction data processing'); Log::warning('No period provided for ATM transaction data processing');
return; return;
} }
$stats = $this->processPeriods(); // Skip the parameter folder
if ($this->period === self::PARAMETER_FOLDER) {
Log::info("Skipping " . self::PARAMETER_FOLDER . " folder");
return;
}
$stats = $this->processPeriodFile();
Log::info("ProcessAtmTransactionJob completed. Total processed: {$stats['processed']}, Total errors: {$stats['errors']}"); Log::info("ProcessAtmTransactionJob completed. Total processed: {$stats['processed']}, Total errors: {$stats['errors']}");
} catch (Exception $e) { } catch (Exception $e) {
@@ -75,40 +82,15 @@
} }
} }
/**
* Process all periods and return statistics
*/
private function processPeriods(): array
{
$disk = Storage::disk(self::DISK_NAME);
$processedCount = 0;
$errorCount = 0;
foreach ($this->periods as $period) {
// Skip the parameter folder
if ($period === self::PARAMETER_FOLDER) {
Log::info("Skipping " . self::PARAMETER_FOLDER . " folder");
continue;
}
$result = $this->processPeriodFile($disk, $period);
$processedCount += $result['processed'];
$errorCount += $result['errors'];
}
return [
'processed' => $processedCount,
'errors' => $errorCount
];
}
/** /**
* Process a single period file * Process a single period file
*/ */
private function processPeriodFile($disk, string $period): array private function processPeriodFile()
: array
{ {
$filename = $period . self::FILE_EXTENSION; $disk = Storage::disk(self::DISK_NAME);
$filePath = "$period/$filename"; $filename = $this->period . self::FILE_EXTENSION;
$filePath = "{$this->period}/$filename";
$processedCount = 0; $processedCount = 0;
$errorCount = 0; $errorCount = 0;
@@ -141,7 +123,8 @@
/** /**
* Create a temporary file for processing * Create a temporary file for processing
*/ */
private function createTempFile($disk, string $filePath, string $filename): string private function createTempFile($disk, string $filePath, string $filename)
: string
{ {
$tempFilePath = storage_path("app/temp_$filename"); $tempFilePath = storage_path("app/temp_$filename");
file_put_contents($tempFilePath, $disk->get($filePath)); file_put_contents($tempFilePath, $disk->get($filePath));
@@ -151,7 +134,8 @@
/** /**
* Process a CSV file and import data * Process a CSV file and import data
*/ */
private function processCSVFile(string $tempFilePath, string $originalFilePath): array private function processCSVFile(string $tempFilePath, string $originalFilePath)
: array
{ {
$processedCount = 0; $processedCount = 0;
$errorCount = 0; $errorCount = 0;
@@ -194,7 +178,8 @@
/** /**
* Process a single row from the CSV file * Process a single row from the CSV file
*/ */
private function processRow(array $headerRow, array $row, int $rowCount, string $filePath): array private function processRow(array $headerRow, array $row, int $rowCount, string $filePath)
: array
{ {
// Combine the header row with the data row // Combine the header row with the data row
$rawData = array_combine($headerRow, $row); $rawData = array_combine($headerRow, $row);

100
app/Jobs/ProcessBillDetailDataJob.php
View File

@@ -1,63 +1,62 @@
<?php <?php
namespace Modules\Webstatement\Jobs; namespace Modules\Webstatement\Jobs;
use Illuminate\Bus\Queueable; use Exception;
use Illuminate\Queue\SerializesModels; use Illuminate\Bus\Queueable;
use Illuminate\Queue\InteractsWithQueue; use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Support\Facades\Log; use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Storage; use Illuminate\Support\Facades\Log;
use Modules\Webstatement\Models\TempBillDetail; use Illuminate\Support\Facades\Storage;
use Exception; use Modules\Webstatement\Models\TempBillDetail;
class ProcessBillDetailDataJob implements ShouldQueue class ProcessBillDetailDataJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $periods;
/**
* Create a new job instance.
*/
public function __construct(array $periods = [])
{ {
$this->periods = $periods; use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
}
/** protected string $period;
* Execute the job.
*/
public function handle()
{
try {
set_time_limit(24 * 60 * 60);
$disk = Storage::disk('sftpStatement');
$processedCount = 0;
$errorCount = 0;
if (empty($this->periods)) { /**
Log::warning('No periods provided for bill detail data processing'); * Create a new job instance.
return; */
} public function __construct(string $period = '')
{
$this->period = $period;
}
/**
* Execute the job.
*/
public function handle()
{
try {
set_time_limit(24 * 60 * 60);
$disk = Storage::disk('sftpStatement');
$processedCount = 0;
$errorCount = 0;
if (empty($this->period)) {
Log::warning('No period provided for bill detail data processing');
return;
}
foreach ($this->periods as $period) {
// Skip the _parameter folder // Skip the _parameter folder
if ($period === '_parameter') { if ($this->period === '_parameter') {
Log::info("Skipping _parameter folder"); Log::info("Skipping _parameter folder");
continue; return;
} }
// Construct the filename based on the period folder name // Construct the filename based on the period folder name
$filename = "$period.ST.AA.BILL.DETAILS.csv"; $filename = "{$this->period}.ST.AA.BILL.DETAILS.csv";
$filePath = "$period/$filename"; $filePath = "{$this->period}/$filename";
Log::info("Processing bill detail file: $filePath"); Log::info("Processing bill detail file: $filePath");
if (!$disk->exists($filePath)) { if (!$disk->exists($filePath)) {
Log::warning("File not found: $filePath"); Log::warning("File not found: $filePath");
continue; return;
} }
// Create a temporary local copy of the file // Create a temporary local copy of the file
@@ -67,7 +66,7 @@ class ProcessBillDetailDataJob implements ShouldQueue
$handle = fopen($tempFilePath, "r"); $handle = fopen($tempFilePath, "r");
if ($handle !== false) { if ($handle !== false) {
$headers = (new TempBillDetail())->getFillable(); $headers = (new TempBillDetail())->getFillable();
$rowCount = 0; $rowCount = 0;
while (($row = fgetcsv($handle, 0, "~")) !== false) { while (($row = fgetcsv($handle, 0, "~")) !== false) {
@@ -78,7 +77,7 @@ class ProcessBillDetailDataJob implements ShouldQueue
try { try {
if (isset($data['_id']) && $data['_id'] !== '_id') { if (isset($data['_id']) && $data['_id'] !== '_id') {
TempBillDetail::updateOrCreate( TempBillDetail::updateOrCreate(
['_id' => $data['_id']], // Fixed the syntax error here ['_id' => $data['_id']],
$data $data
); );
$processedCount++; $processedCount++;
@@ -100,13 +99,12 @@ class ProcessBillDetailDataJob implements ShouldQueue
} else { } else {
Log::error("Unable to open file: $filePath"); Log::error("Unable to open file: $filePath");
} }
Log::info("Bill Detail data processing completed. Total processed: $processedCount, Total errors: $errorCount");
} catch (Exception $e) {
Log::error('Error in ProcessBillDetailDataJob: ' . $e->getMessage());
throw $e;
} }
Log::info("Bill Detail data processing completed. Total processed: $processedCount, Total errors: $errorCount");
} catch (Exception $e) {
Log::error('Error in ProcessBillDetailDataJob: ' . $e->getMessage());
throw $e;
} }
} }
}

152
app/Jobs/ProcessCustomerDataJob.php
View File

@@ -1,107 +1,103 @@
<?php <?php
namespace Modules\Webstatement\Jobs; namespace Modules\Webstatement\Jobs;
use Exception; use Exception;
use Illuminate\Bus\Queueable; use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels; use Illuminate\Queue\SerializesModels;
use Modules\Webstatement\Models\Customer; use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Storage;
use Illuminate\Support\Facades\Storage; use Modules\Webstatement\Models\Customer;
class ProcessCustomerDataJob implements ShouldQueue class ProcessCustomerDataJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $periods;
public function __construct(array $periods = [])
{ {
$this->periods = $periods; use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
}
public function handle() protected $period;
{
try {
set_time_limit(24 * 60 * 60);
$disk = Storage::disk('sftpStatement');
$processedCount = 0;
$errorCount = 0;
if (empty($this->periods)) { public function __construct(string $period = '')
Log::warning('No periods provided for customer data processing'); {
return; $this->period = $period;
} }
foreach ($this->periods as $period) { public function handle()
{
try {
set_time_limit(24 * 60 * 60);
$disk = Storage::disk('sftpStatement');
$processedCount = 0;
$errorCount = 0;
if (empty($this->period)) {
Log::warning('No period provided for customer data processing');
return;
}
// Skip the _parameter folder // Skip the _parameter folder
if ($period === '_parameter') { if ($this->period === '_parameter') {
Log::info("Skipping _parameter folder"); Log::info("Skipping _parameter folder");
continue; return;
} }
// Construct the filename based on the period folder name // Construct the filename based on the period folder name
$filename = "$period.ST.CUSTOMER.csv"; $filename = "{$this->period}.ST.CUSTOMER.csv";
$filePath = "$period/$filename"; $filePath = "{$this->period}/$filename";
Log::info("Processing customer file: $filePath"); Log::info("Processing customer file: $filePath");
if (!$disk->exists($filePath)) { if (!$disk->exists($filePath)) {
Log::warning("File not found: $filePath"); Log::warning("File not found: $filePath");
continue; return;
} }
// Create a temporary local copy of the file // Create a temporary local copy of the file
$tempFilePath = storage_path("app/temp_$filename"); $tempFilePath = storage_path("app/temp_$filename");
file_put_contents($tempFilePath, $disk->get($filePath)); file_put_contents($tempFilePath, $disk->get($filePath));
$handle = fopen($tempFilePath, "r"); $handle = fopen($tempFilePath, "r");
if ($handle !== false) { if ($handle !== false) {
$headers = (new Customer())->getFillable(); $headers = (new Customer())->getFillable();
$rowCount = 0; $rowCount = 0;
while (($row = fgetcsv($handle, 0, "~")) !== false) { while (($row = fgetcsv($handle, 0, "~")) !== false) {
$rowCount++; $rowCount++;
if (count($headers) === count($row)) { if (count($headers) === count($row)) {
$data = array_combine($headers, $row); $data = array_combine($headers, $row);
try { try {
if ($data['customer_code'] !== 'customer_code') { if ($data['customer_code'] !== 'customer_code') {
$customer = Customer::firstOrNew(['customer_code' => $data['customer_code']]); $customer = Customer::firstOrNew(['customer_code' => $data['customer_code']]);
$customer->fill($data); $customer->fill($data);
$customer->save(); $customer->save();
$processedCount++; $processedCount++;
}
} catch (Exception $e) {
$errorCount++;
Log::error("Error processing Customer at row $rowCount in $filePath: " . $e->getMessage());
} }
} else { } catch (Exception $e) {
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row)); $errorCount++;
Log::error("Error processing Customer at row $rowCount in $filePath: " . $e->getMessage());
} }
} else {
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row));
} }
fclose($handle);
Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors.");
// Clean up the temporary file
unlink($tempFilePath);
} else {
Log::error("Unable to open file: $filePath");
} }
fclose($handle);
Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors.");
// Clean up the temporary file
unlink($tempFilePath);
} else {
Log::error("Unable to open file: $filePath");
}
Log::info("Customer data processing completed. Total processed: $processedCount, Total errors: $errorCount");
} catch (Exception $e) {
Log::error('Error in ProcessCustomerDataJob: ' . $e->getMessage());
throw $e;
} }
Log::info("Customer data processing completed. Total processed: $processedCount, Total errors: $errorCount");
} catch (Exception $e) {
Log::error('Error in ProcessCustomerDataJob: ' . $e->getMessage());
throw $e;
} }
} }
}

238
app/Jobs/ProcessDataCaptureDataJob.php
View File

@@ -16,15 +16,15 @@
{ {
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $periods; protected $period;
protected $filename; protected $filename;
/** /**
* Create a new job instance. * Create a new job instance.
*/ */
public function __construct(array $periods = [], string $filename = "ST.DATA.CAPTURE.csv") public function __construct(string $period = '', string $filename = "ST.DATA.CAPTURE.csv")
{ {
$this->periods = $periods; $this->period = $period;
$this->filename = $filename; $this->filename = $filename;
} }
@@ -40,139 +40,137 @@
$processedCount = 0; $processedCount = 0;
$errorCount = 0; $errorCount = 0;
if (empty($this->periods)) { if (empty($this->period)) {
Log::warning('No periods provided for data capture processing'); Log::warning('No period provided for data capture processing');
return; return;
} }
foreach ($this->periods as $period) { // Skip the _parameter folder
// Skip the _parameter folder if ($this->period === '_parameter') {
if ($period === '_parameter') { Log::info("Skipping _parameter folder");
Log::info("Skipping _parameter folder"); return;
continue; }
}
// Construct the filepath based on the period folder name // Construct the filepath based on the period folder name
$fileName = "$period.$this->filename"; $fileName = "{$this->period}.{$this->filename}";
$filePath = "$period/$fileName"; $filePath = "{$this->period}/$fileName";
Log::info("Processing data capture file: $filePath"); Log::info("Processing data capture file: $filePath");
if (!$disk->exists($filePath)) { if (!$disk->exists($filePath)) {
Log::warning("File not found: $filePath"); Log::warning("File not found: $filePath");
continue; return;
} }
// Create a temporary local copy of the file // Create a temporary local copy of the file
$tempFilePath = storage_path("app/temp_{$this->filename}"); $tempFilePath = storage_path("app/temp_{$this->filename}");
file_put_contents($tempFilePath, $disk->get($filePath)); file_put_contents($tempFilePath, $disk->get($filePath));
$handle = fopen($tempFilePath, "r"); $handle = fopen($tempFilePath, "r");
if ($handle !== false) { if ($handle !== false) {
// CSV headers from the file // CSV headers from the file
$csvHeaders = [ $csvHeaders = [
'id', 'id',
'account_number', 'account_number',
'sign', 'sign',
'amount_lcy', 'amount_lcy',
'transaction_code', 'transaction_code',
'their_reference', 'their_reference',
'narrative', 'narrative',
'pl_category', 'pl_category',
'customer_id', 'customer_id',
'account_officer', 'account_officer',
'product_category', 'product_category',
'value_date', 'value_date',
'currency', 'currency',
'amount_fcy', 'amount_fcy',
'exchange_rate', 'exchange_rate',
'neg_ref_no', 'neg_ref_no',
'position_type', 'position_type',
'our_reference', 'our_reference',
'reversal_marker', 'reversal_marker',
'exposure_date', 'exposure_date',
'currency_market', 'currency_market',
'iblc_country', 'iblc_country',
'last_version', 'last_version',
'otor_version', 'otor_version',
'department_code', 'department_code',
'dealer_desk', 'dealer_desk',
'bank_sort_cde', 'bank_sort_cde',
'cheque_number', 'cheque_number',
'accounting_date', 'accounting_date',
'contingent_acct', 'contingent_acct',
'cheq_type', 'cheq_type',
'tfs_reference', 'tfs_reference',
'accounting_company', 'accounting_company',
'stmt_no', 'stmt_no',
'curr_no', 'curr_no',
'inputter', 'inputter',
'authoriser', 'authoriser',
'co_code', 'co_code',
'date_time' 'date_time'
]; ];
$rowCount = 0; $rowCount = 0;
while (($row = fgetcsv($handle, 0, "~")) !== false) { while (($row = fgetcsv($handle, 0, "~")) !== false) {
$rowCount++; $rowCount++;
// Skip header row if it exists // Skip header row if it exists
if ($rowCount === 1 && strtolower($row[0]) === 'id') { if ($rowCount === 1 && strtolower($row[0]) === 'id') {
continue; continue;
}
if (count($csvHeaders) === count($row)) {
$data = array_combine($csvHeaders, $row);
try {
// Format dates if they exist
foreach (['value_date', 'exposure_date', 'accounting_date'] as $dateField) {
if (!empty($data[$dateField])) {
try {
$data[$dateField] = date('Y-m-d', strtotime($data[$dateField]));
} catch (Exception $e) {
// If date parsing fails, keep the original value
Log::warning("Failed to parse date for $dateField: {$data[$dateField]}");
}
}
}
// Format datetime if it exists
if (!empty($data['date_time'])) {
try {
$data['date_time'] = date('Y-m-d H:i:s', strtotime($data['date_time']));
} catch (Exception $e) {
// If datetime parsing fails, keep the original value
Log::warning("Failed to parse datetime for date_time: {$data['date_time']}");
}
}
if (!empty($data['id'])) {
DataCapture::updateOrCreate(
['id' => $data['id']],
$data
);
$processedCount++;
}
} catch (Exception $e) {
$errorCount++;
Log::error("Error processing Data Capture at row $rowCount in $filePath: " . $e->getMessage());
}
} else {
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($csvHeaders) . ", Got: " . count($row));
}
} }
fclose($handle); if (count($csvHeaders) === count($row)) {
Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors."); $data = array_combine($csvHeaders, $row);
// Clean up the temporary file try {
unlink($tempFilePath); // Format dates if they exist
} else { foreach (['value_date', 'exposure_date', 'accounting_date'] as $dateField) {
Log::error("Unable to open file: $filePath"); if (!empty($data[$dateField])) {
try {
$data[$dateField] = date('Y-m-d', strtotime($data[$dateField]));
} catch (Exception $e) {
// If date parsing fails, keep the original value
Log::warning("Failed to parse date for $dateField: {$data[$dateField]}");
}
}
}
// Format datetime if it exists
if (!empty($data['date_time'])) {
try {
$data['date_time'] = date('Y-m-d H:i:s', strtotime($data['date_time']));
} catch (Exception $e) {
// If datetime parsing fails, keep the original value
Log::warning("Failed to parse datetime for date_time: {$data['date_time']}");
}
}
if (!empty($data['id'])) {
DataCapture::updateOrCreate(
['id' => $data['id']],
$data
);
$processedCount++;
}
} catch (Exception $e) {
$errorCount++;
Log::error("Error processing Data Capture at row $rowCount in $filePath: " . $e->getMessage());
}
} else {
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($csvHeaders) . ", Got: " . count($row));
}
} }
fclose($handle);
Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors.");
// Clean up the temporary file
unlink($tempFilePath);
} else {
Log::error("Unable to open file: $filePath");
} }
Log::info("Data capture processing completed. Total processed: $processedCount, Total errors: $errorCount"); Log::info("Data capture processing completed. Total processed: $processedCount, Total errors: $errorCount");

119
app/Jobs/ProcessFundsTransferDataJob.php
View File

@@ -16,95 +16,94 @@
{ {
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $periods; protected $period;
/** /**
* Create a new job instance. * Create a new job instance.
*/ */
public function __construct(array $periods = []) public function __construct(string $period = '')
{ {
$this->periods = $periods; $this->period = $period;
} }
/** /**
* Execute the job. * Execute the job.
*/ */
public function handle(): void public function handle()
: void
{ {
try { try {
set_time_limit(24 * 60 * 60); set_time_limit(24 * 60 * 60);
$disk = Storage::disk('sftpStatement'); $disk = Storage::disk('sftpStatement');
$processedCount = 0; $processedCount = 0;
$errorCount = 0; $errorCount = 0;
if (empty($this->periods)) { if (empty($this->period)) {
Log::warning('No periods provided for funds transfer data processing'); Log::warning('No period provided for funds transfer data processing');
return; return;
} }
foreach ($this->periods as $period) { // Skip the _parameter folder
// Skip the _parameter folder if ($this->period === '_parameter') {
if ($period === '_parameter') { Log::info("Skipping _parameter folder");
Log::info("Skipping _parameter folder"); return;
continue; }
}
// Construct the filename based on the period folder name // Construct the filename based on the period folder name
$filename = "$period.ST.FUNDS.TRANSFER.csv"; $filename = "{$this->period}.ST.FUNDS.TRANSFER.csv";
$filePath = "$period/$filename"; $filePath = "{$this->period}/$filename";
Log::info("Processing funds transfer file: $filePath"); Log::info("Processing funds transfer file: $filePath");
if (!$disk->exists($filePath)) { if (!$disk->exists($filePath)) {
Log::warning("File not found: $filePath"); Log::warning("File not found: $filePath");
continue; return;
} }
// Create a temporary local copy of the file // Create a temporary local copy of the file
$tempFilePath = storage_path("app/temp_$filename"); $tempFilePath = storage_path("app/temp_$filename");
file_put_contents($tempFilePath, $disk->get($filePath)); file_put_contents($tempFilePath, $disk->get($filePath));
$handle = fopen($tempFilePath, "r"); $handle = fopen($tempFilePath, "r");
if ($handle !== false) { if ($handle !== false) {
$headers = (new TempFundsTransfer())->getFillable(); $headers = (new TempFundsTransfer())->getFillable();
$rowCount = 0; $rowCount = 0;
while (($row = fgetcsv($handle, 0, "~")) !== false) { while (($row = fgetcsv($handle, 0, "~")) !== false) {
$rowCount++; $rowCount++;
// Handle case where row has more columns than headers // Handle case where row has more columns than headers
if (count($row) > count($headers)) { if (count($row) > count($headers)) {
$row = array_slice($row, 0, count($headers)); $row = array_slice($row, 0, count($headers));
}
if (count($headers) === count($row)) {
$data = array_combine($headers, $row);
try {
if (isset($data['_id']) && $data['_id'] !== '_id') {
TempFundsTransfer::updateOrCreate(
['_id' => $data['_id']],
$data
);
$processedCount++;
}
} catch (Exception $e) {
$errorCount++;
Log::error("Error processing Funds Transfer at row $rowCount in $filePath: " . $e->getMessage());
}
} else {
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row));
}
} }
fclose($handle); if (count($headers) === count($row)) {
Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors."); $data = array_combine($headers, $row);
try {
// Clean up the temporary file if (isset($data['_id']) && $data['_id'] !== '_id') {
unlink($tempFilePath); TempFundsTransfer::updateOrCreate(
} else { ['_id' => $data['_id']],
Log::error("Unable to open file: $filePath"); $data
);
$processedCount++;
}
} catch (Exception $e) {
$errorCount++;
Log::error("Error processing Funds Transfer at row $rowCount in $filePath: " . $e->getMessage());
}
} else {
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row));
}
} }
fclose($handle);
Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors.");
// Clean up the temporary file
unlink($tempFilePath);
} else {
Log::error("Unable to open file: $filePath");
} }
Log::info("Funds Transfer data processing completed. Total processed: $processedCount, Total errors: $errorCount"); Log::info("Funds Transfer data processing completed. Total processed: $processedCount, Total errors: $errorCount");

116
app/Jobs/ProcessStmtEntryDataJob.php
View File

@@ -11,101 +11,99 @@
use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage; use Illuminate\Support\Facades\Storage;
use Modules\Webstatement\Models\StmtEntry; use Modules\Webstatement\Models\StmtEntry;
use Modules\Webstatement\Models\TempStmtEntry;
class ProcessStmtEntryDataJob implements ShouldQueue class ProcessStmtEntryDataJob implements ShouldQueue
{ {
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $periods; protected $period;
/** /**
* Create a new job instance. * Create a new job instance.
*/ */
public function __construct(array $periods = []) public function __construct(string $period = '')
{ {
$this->periods = $periods; $this->period = $period;
} }
/** /**
* Execute the job. * Execute the job.
*/ */
public function handle(): void public function handle()
: void
{ {
try { try {
set_time_limit(24 * 60 * 60); set_time_limit(24 * 60 * 60);
$disk = Storage::disk('sftpStatement'); $disk = Storage::disk('sftpStatement');
$processedCount = 0; $processedCount = 0;
$errorCount = 0; $errorCount = 0;
if (empty($this->periods)) { if (empty($this->period)) {
Log::warning('No periods provided for statement entry data processing'); Log::warning('No period provided for statement entry data processing');
return; return;
} }
foreach ($this->periods as $period) { // Skip the _parameter folder
// Skip the _parameter folder if ($this->period === '_parameter') {
if ($period === '_parameter') { Log::info("Skipping _parameter folder");
Log::info("Skipping _parameter folder"); return;
continue; }
}
// Construct the filename based on the period folder name // Construct the filename based on the period folder name
$filename = "$period.ST.STMT.ENTRY.csv"; $filename = "{$this->period}.ST.STMT.ENTRY.csv";
$filePath = "$period/$filename"; $filePath = "{$this->period}/$filename";
Log::info("Processing statement entry file: $filePath"); Log::info("Processing statement entry file: $filePath");
if (!$disk->exists($filePath)) { if (!$disk->exists($filePath)) {
Log::warning("File not found: $filePath"); Log::warning("File not found: $filePath");
continue; return;
} }
// Create a temporary local copy of the file // Create a temporary local copy of the file
$tempFilePath = storage_path("app/temp_$filename"); $tempFilePath = storage_path("app/temp_$filename");
file_put_contents($tempFilePath, $disk->get($filePath)); file_put_contents($tempFilePath, $disk->get($filePath));
$handle = fopen($tempFilePath, "r"); $handle = fopen($tempFilePath, "r");
if ($handle !== false) { if ($handle !== false) {
$headers = (new StmtEntry())->getFillable(); $headers = (new StmtEntry())->getFillable();
$rowCount = 0; $rowCount = 0;
while (($row = fgetcsv($handle, 0, "~")) !== false) { while (($row = fgetcsv($handle, 0, "~")) !== false) {
$rowCount++; $rowCount++;
if (count($headers) === count($row)) { if (count($headers) === count($row)) {
$data = array_combine($headers, $row); $data = array_combine($headers, $row);
try { try {
if ($data['stmt_entry_id'] !== 'stmt_entry_id') { if ($data['stmt_entry_id'] !== 'stmt_entry_id') {
// Bersihkan trans_reference dari \\BNK jika ada // Bersihkan trans_reference dari \\BNK jika ada
if (isset($data['trans_reference'])) { if (isset($data['trans_reference'])) {
$data['trans_reference'] = str_replace('\\BNK', '', $data['trans_reference']); $data['trans_reference'] = str_replace('\\BNK', '', $data['trans_reference']);
}
StmtEntry::updateOrCreate(
['stmt_entry_id' => $data['stmt_entry_id']],
$data
);
$processedCount++;
} }
} catch (Exception $e) {
$errorCount++; StmtEntry::updateOrCreate(
Log::error("Error processing Statement Entry at row $rowCount in $filePath: " . $e->getMessage()); ['stmt_entry_id' => $data['stmt_entry_id']],
$data
);
$processedCount++;
} }
} else { } catch (Exception $e) {
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row)); $errorCount++;
Log::error("Error processing Statement Entry at row $rowCount in $filePath: " . $e->getMessage());
} }
} else {
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row));
} }
fclose($handle);
Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors.");
// Clean up the temporary file
unlink($tempFilePath);
} else {
Log::error("Unable to open file: $filePath");
} }
fclose($handle);
Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors.");
// Clean up the temporary file
unlink($tempFilePath);
} else {
Log::error("Unable to open file: $filePath");
} }
Log::info("Statement Entry data processing completed. Total processed: $processedCount, Total errors: $errorCount"); Log::info("Statement Entry data processing completed. Total processed: $processedCount, Total errors: $errorCount");

61
app/Jobs/ProcessTellerDataJob.php
View File

@@ -19,10 +19,10 @@
private const PARAMETER_FOLDER = '_parameter'; private const PARAMETER_FOLDER = '_parameter';
// Konstanta untuk nilai-nilai statis // Konstanta untuk nilai-nilai statis
private const FILE_EXTENSION = '.ST.TELLER.csv'; private const FILE_EXTENSION = '.ST.TELLER.csv';
private const CSV_DELIMITER = '~'; private const CSV_DELIMITER = '~';
private const DISK_NAME = 'sftpStatement'; private const DISK_NAME = 'sftpStatement';
private const HEADER_MAP = [ private const HEADER_MAP = [
'id' => 'id_teller', 'id' => 'id_teller',
'account_1' => 'account_1', 'account_1' => 'account_1',
'currency_1' => 'currency_1', 'currency_1' => 'currency_1',
@@ -129,14 +129,14 @@
]; ];
// Pemetaan bidang header ke kolom model // Pemetaan bidang header ke kolom model
protected array $periods; protected string $period;
/** /**
* Create a new job instance. * Create a new job instance.
*/ */
public function __construct(array $periods = []) public function __construct(string $period = '')
{ {
$this->periods = $periods; $this->period = $period;
} }
/** /**
@@ -148,12 +148,18 @@
try { try {
set_time_limit(24 * 60 * 60); set_time_limit(24 * 60 * 60);
if (empty($this->periods)) { if (empty($this->period)) {
Log::warning('No periods provided for teller data processing'); Log::warning('No period provided for teller data processing');
return; return;
} }
$stats = $this->processPeriods(); // Skip the parameter folder
if ($this->period === self::PARAMETER_FOLDER) {
Log::info("Skipping " . self::PARAMETER_FOLDER . " folder");
return;
}
$stats = $this->processPeriodFile();
Log::info("ProcessTellerDataJob completed. Total processed: {$stats['processed']}, Total errors: {$stats['errors']}"); Log::info("ProcessTellerDataJob completed. Total processed: {$stats['processed']}, Total errors: {$stats['errors']}");
} catch (Exception $e) { } catch (Exception $e) {
@@ -163,41 +169,14 @@
} }
/** /**
* Process all periods and return statistics * Process a single period file
*/ */
private function processPeriods() private function processPeriodFile()
: array : array
{ {
$disk = Storage::disk(self::DISK_NAME); $disk = Storage::disk(self::DISK_NAME);
$processedCount = 0; $filename = $this->period . self::FILE_EXTENSION;
$errorCount = 0; $filePath = "{$this->period}/$filename";
foreach ($this->periods as $period) {
// Skip the parameter folder
if ($period === self::PARAMETER_FOLDER) {
Log::info("Skipping " . self::PARAMETER_FOLDER . " folder");
continue;
}
$result = $this->processPeriodFile($disk, $period);
$processedCount += $result['processed'];
$errorCount += $result['errors'];
}
return [
'processed' => $processedCount,
'errors' => $errorCount
];
}
/**
* Process a single period file
*/
private function processPeriodFile($disk, string $period)
: array
{
$filename = $period . self::FILE_EXTENSION;
$filePath = "$period/$filename";
$processedCount = 0; $processedCount = 0;
$errorCount = 0; $errorCount = 0;