daengdeni/webstatement
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(webstatement): optimalkan proses migrasi data SFTP

Browse Source 
- Menambahkan parameter `$periods` untuk mendukung proses dinamis berdasarkan folder periode dalam SFTP.
- Memperkenalkan penggunaan `Storage::disk('sftpStatement')` untuk membaca file dari SFTP.
- Menyortir folder periode secara menurun berdasarkan tanggal.
- Memproses tiap entitas data (Customer, Account, Arrangement, Bill Details, dll.) berdasarkan file dalam folder periode.
- Menambah log untuk mencatat jumlah proses yang berhasil, baris error, dan peringatan tentang ketidaksesuaian jumlah kolom.
- Menyisipkan mekanisme untuk mengabaikan folder `_parameter`.
- Mengubah pendekatan dari membaca file lokal menjadi menggunakan file sementara (temp).
- Memperbaiki bug pada metode yang sebelumnya menggunakan jalur file statis.
- Memastikan penanganan file CSV yang lebih fleksibel, termasuk kasus di mana jumlah kolom dalam baris melebihi ekspektasi.
- Menambah error handling untuk mencatat baris-baris bermasalah dan menghindari interupsi proses.

Signed-off-by: Daeng Deni Mardaeni <ddeni05@gmail.com>
This commit is contained in:
Daeng Deni Mardaeni
2025-05-20 16:44:59 +07:00
parent 7bb320d718
commit c442b27dad
11 changed files with 748 additions and 297 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
app/Http/Controllers/MigrasiController.php
View File

@@ -4,6 +4,7 @@ namespace Modules\Webstatement\Http\Controllers;
use App\Http\Controllers\Controller;
use Exception;
use Illuminate\Support\Facades\Storage;
use Log;
use Modules\Webstatement\Jobs\ProcessAccountDataJob;
use Modules\Webstatement\Jobs\ProcessArrangementDataJob;
@@ -18,49 +19,54 @@ use Modules\Webstatement\Jobs\ProcessTransactionDataJob;
class MigrasiController extends Controller
{
public function processArrangementData()
public function processArrangementData($periods)
{
try {
ProcessArrangementDataJob::dispatch();
ProcessArrangementDataJob::dispatch($periods);
return response()->json(['message' => 'Data Arrangement processing job has been successfully']);
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function processCustomerData()
public function processCustomerData($periods)
{
$filePath = storage_path('app/20240901.ST.CUSTOMER.csv'); // Adjust this path as needed
try {
ProcessCustomerDataJob::dispatch();
return response()->json(['message' => 'Data Customer processing job has been successfully']);
// Pass the periods to the job for processing
ProcessCustomerDataJob::dispatch($periods);
return response()->json([
'message' => 'Data Customer processing job has been successfully queued',
'periods' => $periods
]);
} catch (Exception $e) {
Log::error('Error in processCustomerData: ' . $e->getMessage());
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function processBillDetailData()
public function processBillDetailData($periods)
{
try {
ProcessBillDetailDataJob::dispatch();
ProcessBillDetailDataJob::dispatch($periods);
return response()->json(['message' => 'Data Bill Details processing job has been successfully']);
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function processAccountData(){
public function processAccountData($periods){
try{
ProcessAccountDataJob::dispatch();
ProcessAccountDataJob::dispatch($periods);
return response()->json(['message' => 'Data Account processing job has been successfully']);
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function processTransactionData(){
public function processTransactionData($periods){
try{
ProcessTransactionDataJob::dispatch();
ProcessTransactionDataJob::dispatch($periods);
Log::info('Data Transaction processing job has been successfully');
return response()->json(['message' => 'Data Transaction processing job has been successfully']);
} catch (Exception $e) {
@@ -68,37 +74,37 @@ class MigrasiController extends Controller
}
}
public function processFundsTransferData(){
public function processFundsTransferData($periods){
try{
ProcessFundsTransferDataJob::dispatch();
ProcessFundsTransferDataJob::dispatch($periods);
return response()->json(['message' => 'Data Funds Transfer processing job has been successfully']);
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function processStmtNarrParamData()
public function processStmtNarrParamData($periods)
{
try {
ProcessStmtNarrParamDataJob::dispatch();
ProcessStmtNarrParamDataJob::dispatch($periods);
return response()->json(['message' => 'Data TempStmtNarrParam processing job has been successfully']);
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function processStmtNarrFormatData(){
public function processStmtNarrFormatData($periods){
try {
ProcessStmtNarrFormatDataJob::dispatch();
ProcessStmtNarrFormatDataJob::dispatch($periods);
return response()->json(['message' => 'Data TempStmtNarrFormat processing job has been successfully']);
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
}
}
public function processStmtEntryData(){
public function processStmtEntryData($periods){
try {
ProcessStmtEntryDataJob::dispatch();
ProcessStmtEntryDataJob::dispatch($periods);
return response()->json(['message' => 'Data TempStmtEntry processing job has been successfully']);
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
@@ -108,15 +114,34 @@ class MigrasiController extends Controller
public function index()
{
$this->processCustomerData();
$this->processAccountData();
$this->processArrangementData();
$this->processBillDetailData();
$this->processTransactionData();
$this->processFundsTransferData();
$this->processStmtNarrParamData();
$this->processStmtNarrFormatData();
$this->processStmtEntryData();
$disk = Storage::disk('sftpStatement');
// Get all directories (periods) in the SFTP disk
$allDirectories = $disk->directories();
// Filter out the _parameter folder
$periods = array_filter($allDirectories, function($dir) {
return $dir !== '_parameter';
});
// Sort periods by date (descending)
usort($periods, function($a, $b) {
return strcmp($b, $a); // Reverse comparison for descending order
});
if (empty($periods)) {
return response()->json(['message' => 'No valid period folders found in SFTP storage'], 404);
}
$this->processCustomerData($periods);
$this->processAccountData($periods);
$this->processArrangementData($periods);
$this->processBillDetailData($periods);
$this->processTransactionData($periods);
$this->processFundsTransferData($periods);
$this->processStmtNarrParamData($periods);
$this->processStmtNarrFormatData($periods);
$this->processStmtEntryData($periods);
return response()->json(['message' => 'Data processing job has been successfully']);
}

113
app/Jobs/ProcessAccountDataJob.php
View File

@@ -8,72 +8,113 @@
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Log;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
use Modules\Webstatement\Models\Account;
class ProcessAccountDataJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $periods;
/**
* Create a new job instance.
*/
public function __construct()
public function __construct(array $periods = [])
{
//
$this->periods = $periods;
}
/**
* Execute the job.
*/
public function handle()
: void
public function handle(): void
{
$filePath = storage_path('app/20250207.ST.ACCOUNT.csv'); // Use Storage facade
try {
if (!file_exists($filePath)) {
throw new Exception("File not found: {$filePath}");
set_time_limit(24 * 60 * 60);
$disk = Storage::disk('sftpStatement');
$processedCount = 0;
$errorCount = 0;
if (empty($this->periods)) {
Log::warning('No periods provided for account data processing');
return;
}
set_time_limit(24 * 60 * 60);
foreach ($this->periods as $period) {
// Skip the _parameter folder
if ($period === '_parameter') {
Log::info("Skipping _parameter folder");
continue;
}
$handle = fopen($filePath, "r");
if ($handle !== false) {
// Construct the filename based on the period folder name
$filename = "$period.ST.ACCOUNT.csv";
$filePath = "$period/$filename";
// Use the header row to determine the order of columns
$headers = (new Account())->getFillable();
Log::info('Headers: ' . implode(", ", $headers));
$rowCount = 0;
while (($row = fgetcsv($handle, 0, "~")) !== false) {
if (count($headers) === count($row)) {
$data = array_combine($headers, $row);
Log::info("Processing account file: $filePath");
// Check if start_year_bal is empty and set it to 0 if so
if (empty($data['start_year_bal']) || $data['start_year_bal'] ="" || $data['start_year_bal']== null) {
$data['start_year_bal'] = 0;
}
if (!$disk->exists($filePath)) {
Log::warning("File not found: $filePath");
continue;
}
if (empty($data['closure_date']) || $data['closure_date'] ="" || $data['closure_date']== null) {
$data['closure_date'] = null;
}
// Create a temporary local copy of the file
$tempFilePath = storage_path("app/temp_$filename");
file_put_contents($tempFilePath, $disk->get($filePath));
try {
if( $data['account_number'] !== 'account_number') {
// Use firstOrNew instead of updateOrCreate
$account = Account::firstOrNew(['account_number' => $data['account_number']]);
$account->fill($data);
$account->save();
$handle = fopen($tempFilePath, "r");
if ($handle !== false) {
$headers = (new Account())->getFillable();
Log::info('Headers: ' . implode(", ", $headers));
$rowCount = 0;
while (($row = fgetcsv($handle, 0, "~")) !== false) {
$rowCount++;
if (count($headers) === count($row)) {
$data = array_combine($headers, $row);
// Check if start_year_bal is empty and set it to 0 if so
if (empty($data['start_year_bal']) || $data['start_year_bal'] == "" || $data['start_year_bal'] == null) {
$data['start_year_bal'] = 0;
}
} catch (Exception $e) {
Log::error('Error processing Account: ' . $e->getMessage());
if (empty($data['closure_date']) || $data['closure_date'] == "" || $data['closure_date'] == null) {
$data['closure_date'] = null;
}
try {
if ($data['account_number'] !== 'account_number') {
// Use firstOrNew instead of updateOrCreate
$account = Account::firstOrNew(['account_number' => $data['account_number']]);
$account->fill($data);
$account->save();
$processedCount++;
}
} catch (Exception $e) {
$errorCount++;
Log::error("Error processing Account at row $rowCount in $filePath: " . $e->getMessage());
}
} else {
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row));
}
}
fclose($handle);
Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors.");
// Clean up the temporary file
unlink($tempFilePath);
} else {
Log::error("Unable to open file: $filePath");
}
fclose($handle);
} else {
throw new Exception("Unable to open file: {$filePath}");
}
Log::info("Account data processing completed. Total processed: $processedCount, Total errors: $errorCount");
} catch (Exception $e) {
Log::error('Error in ProcessAccountDataJob: ' . $e->getMessage());
throw $e;

93
app/Jobs/ProcessArrangementDataJob.php
View File

@@ -8,55 +8,102 @@
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Log;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
use Modules\Webstatement\Models\TempArrangement;
class ProcessArrangementDataJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $periods;
/**
* Create a new job instance.
*/
public function __construct()
public function __construct(array $periods = [])
{
//
$this->periods = $periods;
}
/**
* Execute the job.
*/
public function handle()
: void
public function handle(): void
{
$filePath = storage_path('app/20250207.ST.AA.ARRANGEMENT.csv'); // Adjust this path as needed
try {
if (!file_exists($filePath)) {
throw new Exception("File not found: $filePath");
set_time_limit(24 * 60 * 60);
$disk = Storage::disk('sftpStatement');
$processedCount = 0;
$errorCount = 0;
if (empty($this->periods)) {
Log::warning('No periods provided for arrangement data processing');
return;
}
set_time_limit(24 * 60 * 60);
foreach ($this->periods as $period) {
// Skip the _parameter folder
if ($period === '_parameter') {
Log::info("Skipping _parameter folder");
continue;
}
$handle = fopen($filePath, "r");
// Construct the filename based on the period folder name
$filename = "$period.ST.AA.ARRANGEMENT.csv";
$filePath = "$period/$filename";
if ($handle !== false) {
$headers = (new TempArrangement())->getFillable();
Log::info("Processing arrangement file: $filePath");
while (($row = fgetcsv($handle, 0, "~")) !== false) {
if (count($headers) === count($row)) {
$data = array_combine($headers, $row);
if($data['arrangement_id']!== 'arrangement_id') {
TempArrangement::updateOrCreate(
['arrangement_id' => $data['arrangement_id']], // key to find existing record
$data // data to update or create
);
if (!$disk->exists($filePath)) {
Log::warning("File not found: $filePath");
continue;
}
// Create a temporary local copy of the file
$tempFilePath = storage_path("app/temp_$filename");
file_put_contents($tempFilePath, $disk->get($filePath));
$handle = fopen($tempFilePath, "r");
if ($handle !== false) {
$headers = (new TempArrangement())->getFillable();
$rowCount = 0;
while (($row = fgetcsv($handle, 0, "~")) !== false) {
$rowCount++;
if (count($headers) === count($row)) {
$data = array_combine($headers, $row);
try {
if ($data['arrangement_id'] !== 'arrangement_id') {
TempArrangement::updateOrCreate(
['arrangement_id' => $data['arrangement_id']], // key to find existing record
$data // data to update or create
);
$processedCount++;
}
} catch (Exception $e) {
$errorCount++;
Log::error("Error processing Arrangement at row $rowCount in $filePath: " . $e->getMessage());
}
} else {
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row));
}
}
fclose($handle);
Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors.");
// Clean up the temporary file
unlink($tempFilePath);
} else {
Log::error("Unable to open file: $filePath");
}
fclose($handle);
} else {
throw new Exception("Unable to open file: {$filePath}");
}
Log::info("Arrangement data processing completed. Total processed: $processedCount, Total errors: $errorCount");
} catch (Exception $e) {
Log::error('Error in ProcessArrangementDataJob: ' . $e->getMessage());
throw $e;

91
app/Jobs/ProcessBillDetailDataJob.php
View File

@@ -7,18 +7,23 @@ use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
use Modules\Webstatement\Models\TempBillDetail;
use Exception;
class ProcessBillDetailDataJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $periods;
/**
* Create a new job instance.
*/
public function __construct()
public function __construct(array $periods = [])
{
//
$this->periods = $periods;
}
/**
@@ -26,35 +31,81 @@ class ProcessBillDetailDataJob implements ShouldQueue
*/
public function handle()
{
$filePath = storage_path('app/20240901.ST.AA.BILL.DETAILS.csv');
try {
if (!file_exists($filePath)) {
throw new Exception("File not found: $filePath");
set_time_limit(24 * 60 * 60);
$disk = Storage::disk('sftpStatement');
$processedCount = 0;
$errorCount = 0;
if (empty($this->periods)) {
Log::warning('No periods provided for bill detail data processing');
return;
}
set_time_limit(24 * 60 * 60);
foreach ($this->periods as $period) {
// Skip the _parameter folder
if ($period === '_parameter') {
Log::info("Skipping _parameter folder");
continue;
}
$handle = fopen($filePath, "r");
// Construct the filename based on the period folder name
$filename = "$period.ST.AA.BILL.DETAILS.csv";
$filePath = "$period/$filename";
if ($handle !== false) {
$headers = (new TempBillDetail())->getFillable();
while (($row = fgetcsv($handle, 0, ";")) !== false) {
if (count($headers) === count($row)) {
$data = array_combine($headers, $row);
Log::info("Processing bill detail file: $filePath");
try {
TempBillDetail::upadteOrCreate(['_id',$data['_id']],$data);
} catch (Exception $e) {
\Log::error('Error processing bill detail: ' . $e->getMessage());
if (!$disk->exists($filePath)) {
Log::warning("File not found: $filePath");
continue;
}
// Create a temporary local copy of the file
$tempFilePath = storage_path("app/temp_$filename");
file_put_contents($tempFilePath, $disk->get($filePath));
$handle = fopen($tempFilePath, "r");
if ($handle !== false) {
$headers = (new TempBillDetail())->getFillable();
$rowCount = 0;
while (($row = fgetcsv($handle, 0, ";")) !== false) {
$rowCount++;
if (count($headers) === count($row)) {
$data = array_combine($headers, $row);
try {
if (isset($data['_id']) && $data['_id'] !== '_id') {
TempBillDetail::updateOrCreate(
['_id' => $data['_id']], // Fixed the syntax error here
$data
);
$processedCount++;
}
} catch (Exception $e) {
$errorCount++;
Log::error("Error processing Bill Detail at row $rowCount in $filePath: " . $e->getMessage());
}
} else {
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row));
}
}
fclose($handle);
Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors.");
// Clean up the temporary file
unlink($tempFilePath);
} else {
Log::error("Unable to open file: $filePath");
}
fclose($handle);
} else {
throw new Exception("Unable to open file: {$filePath}");
}
Log::info("Bill Detail data processing completed. Total processed: $processedCount, Total errors: $errorCount");
} catch (Exception $e) {
\Log::error('Error in ProcessBillDetailDataJob: ' . $e->getMessage());
Log::error('Error in ProcessBillDetailDataJob: ' . $e->getMessage());
throw $e;
}
}

103
app/Jobs/ProcessCustomerDataJob.php
View File

@@ -10,57 +10,98 @@ use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Modules\Webstatement\Models\Customer;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
class ProcessCustomerDataJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public function __construct()
{
protected $periods;
public function __construct(array $periods = [])
{
$this->periods = $periods;
}
public function handle()
{
$filePath = storage_path('app/20250207.ST.CUSTOMER.csv'); // Adjust this path as needed
try {
if (!file_exists($filePath)) {
throw new Exception("File not found: $filePath");
}
set_time_limit(24 * 60 * 60);
$disk = Storage::disk('sftpStatement');
$processedCount = 0;
$errorCount = 0;
if (!file_exists($filePath)) {
throw new Exception("File not found: {$filePath}");
if (empty($this->periods)) {
Log::warning('No periods provided for customer data processing');
return;
}
$handle = fopen($filePath, "r");
foreach ($this->periods as $period) {
if ($handle !== false) {
$headers = (new Customer())->getFillable();
while (($row = fgetcsv($handle, 0, "~")) !== false) {
if (count($headers) === count($row)) {
$data = array_combine($headers, $row);
try {
if( $data['customer_code'] !== 'customer_code') {
$customer = Customer::firstOrNew(['customer_code' => $data['customer_code']]);
$customer->fill($data);
$customer->save();
}
}catch (Exception $e) {
Log::error('Error processing Customer: ' . $e->getMessage());
}
}
// Skip the _parameter folder
if ($period === '_parameter') {
Log::info("Skipping _parameter folder");
continue;
}
fclose($handle);
} else {
throw new Exception("Unable to open file: {$filePath}");
// Construct the filename based on the period folder name
$filename = "$period.ST.CUSTOMER.csv";
$filePath = "$period/$filename";
Log::info("Processing customer file: $filePath");
if (!$disk->exists($filePath)) {
Log::warning("File not found: $filePath");
continue;
}
// Create a temporary local copy of the file
$tempFilePath = storage_path("app/temp_$filename");
file_put_contents($tempFilePath, $disk->get($filePath));
$handle = fopen($tempFilePath, "r");
if ($handle !== false) {
$headers = (new Customer())->getFillable();
$rowCount = 0;
while (($row = fgetcsv($handle, 0, "~")) !== false) {
$rowCount++;
if (count($headers) === count($row)) {
$data = array_combine($headers, $row);
try {
if ($data['customer_code'] !== 'customer_code') {
$customer = Customer::firstOrNew(['customer_code' => $data['customer_code']]);
$customer->fill($data);
$customer->save();
$processedCount++;
}
} catch (Exception $e) {
$errorCount++;
Log::error("Error processing Customer at row $rowCount in $filePath: " . $e->getMessage());
}
} else {
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row));
}
}
fclose($handle);
Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors.");
// Clean up the temporary file
unlink($tempFilePath);
} else {
Log::error("Unable to open file: $filePath");
}
}
return response()->json(['message' => 'Data Customer processing job has been successfully']);
Log::info("Customer data processing completed. Total processed: $processedCount, Total errors: $errorCount");
} catch (Exception $e) {
return response()->json(['error' => $e->getMessage()], 500);
Log::error('Error in ProcessCustomerDataJob: ' . $e->getMessage());
throw $e;
}
}
}

102
app/Jobs/ProcessFundsTransferDataJob.php
View File

@@ -8,61 +8,107 @@
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Log;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
use Modules\Webstatement\Models\TempFundsTransfer;
class ProcessFundsTransferDataJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $periods;
/**
* Create a new job instance.
*/
public function __construct()
public function __construct(array $periods = [])
{
//
$this->periods = $periods;
}
/**
* Execute the job.
*/
public function handle()
: void
public function handle(): void
{
$filePath = storage_path('app/20240901.ST.FUNDS.TRANSFER.csv'); // Adjust this path as needed
try {
if (!file_exists($filePath)) {
throw new Exception("File not found: $filePath");
}
set_time_limit(24 * 60 * 60);
$disk = Storage::disk('sftpStatement');
$processedCount = 0;
$errorCount = 0;
if (!file_exists($filePath)) {
throw new Exception("File not found: {$filePath}");
if (empty($this->periods)) {
Log::warning('No periods provided for funds transfer data processing');
return;
}
$handle = fopen($filePath, "r");
foreach ($this->periods as $period) {
// Skip the _parameter folder
if ($period === '_parameter') {
Log::info("Skipping _parameter folder");
continue;
}
if ($handle !== false) {
$headers = (new TempFundsTransfer())->getFillable();
while (($row = fgetcsv($handle, 0, "~")) !== false) {
if (count($row) > count($headers)) {
$row = array_slice($row, 0, count($headers));
}
if (count($headers) === count($row)) {
$data = array_combine($headers, $row);
// Construct the filename based on the period folder name
$filename = "$period.ST.FUNDS.TRANSFER.csv";
$filePath = "$period/$filename";
try {
TempFundsTransfer::updateOrCreate(['_id' => $data['_id']], $data);
} catch (Exception $e) {
Log::error('Error processing funds transfer: ' . $e->getMessage());
Log::info("Processing funds transfer file: $filePath");
if (!$disk->exists($filePath)) {
Log::warning("File not found: $filePath");
continue;
}
// Create a temporary local copy of the file
$tempFilePath = storage_path("app/temp_$filename");
file_put_contents($tempFilePath, $disk->get($filePath));
$handle = fopen($tempFilePath, "r");
if ($handle !== false) {
$headers = (new TempFundsTransfer())->getFillable();
$rowCount = 0;
while (($row = fgetcsv($handle, 0, "~")) !== false) {
$rowCount++;
// Handle case where row has more columns than headers
if (count($row) > count($headers)) {
$row = array_slice($row, 0, count($headers));
}
if (count($headers) === count($row)) {
$data = array_combine($headers, $row);
try {
if (isset($data['_id']) && $data['_id'] !== '_id') {
TempFundsTransfer::updateOrCreate(
['_id' => $data['_id']],
$data
);
$processedCount++;
}
} catch (Exception $e) {
$errorCount++;
Log::error("Error processing Funds Transfer at row $rowCount in $filePath: " . $e->getMessage());
}
} else {
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row));
}
}
fclose($handle);
Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors.");
// Clean up the temporary file
unlink($tempFilePath);
} else {
Log::error("Unable to open file: $filePath");
}
fclose($handle);
} else {
throw new Exception("Unable to open file: {$filePath}");
}
Log::info("Funds Transfer data processing completed. Total processed: $processedCount, Total errors: $errorCount");
} catch (Exception $e) {
Log::error('Error in ProcessFundsTransferDataJob: ' . $e->getMessage());
throw $e;

94
app/Jobs/ProcessStmtEntryDataJob.php
View File

@@ -8,58 +8,102 @@
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Log;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
use Modules\Webstatement\Models\TempStmtEntry;
class ProcessStmtEntryDataJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $periods;
/**
* Create a new job instance.
*/
public function __construct()
public function __construct(array $periods = [])
{
//
$this->periods = $periods;
}
/**
* Execute the job.
*/
public function handle()
: void
public function handle(): void
{
$filePath = storage_path('app/20240901.ST.STMT.ENTRY.csv'); // Adjust this path as needed
try {
if (!file_exists($filePath)) {
throw new Exception("File not found: $filePath");
}
set_time_limit(24 * 60 * 60);
$disk = Storage::disk('sftpStatement');
$processedCount = 0;
$errorCount = 0;
if (!file_exists($filePath)) {
throw new Exception("File not found: {$filePath}");
if (empty($this->periods)) {
Log::warning('No periods provided for statement entry data processing');
return;
}
$handle = fopen($filePath, "r");
foreach ($this->periods as $period) {
// Skip the _parameter folder
if ($period === '_parameter') {
Log::info("Skipping _parameter folder");
continue;
}
if ($handle !== false) {
$headers = (new TempStmtEntry())->getFillable();
while (($row = fgetcsv($handle, 0, "/")) !== false) {
if (count($headers) === count($row)) {
$data = array_combine($headers, $row);
// Construct the filename based on the period folder name
$filename = "$period.ST.STMT.ENTRY.csv";
$filePath = "$period/$filename";
try {
TempStmtEntry::updateOrCreate(['_id' => $data['_id']], $data);
} catch (Exception $e) {
Log::error('Error processing stmt entry: ' . $e->getMessage());
Log::info("Processing statement entry file: $filePath");
if (!$disk->exists($filePath)) {
Log::warning("File not found: $filePath");
continue;
}
// Create a temporary local copy of the file
$tempFilePath = storage_path("app/temp_$filename");
file_put_contents($tempFilePath, $disk->get($filePath));
$handle = fopen($tempFilePath, "r");
if ($handle !== false) {
$headers = (new TempStmtEntry())->getFillable();
$rowCount = 0;
while (($row = fgetcsv($handle, 0, "/")) !== false) {
$rowCount++;
if (count($headers) === count($row)) {
$data = array_combine($headers, $row);
try {
if (isset($data['_id']) && $data['_id'] !== '_id') {
TempStmtEntry::updateOrCreate(
['_id' => $data['_id']],
$data
);
$processedCount++;
}
} catch (Exception $e) {
$errorCount++;
Log::error("Error processing Statement Entry at row $rowCount in $filePath: " . $e->getMessage());
}
} else {
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row));
}
}
fclose($handle);
Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors.");
// Clean up the temporary file
unlink($tempFilePath);
} else {
Log::error("Unable to open file: $filePath");
}
fclose($handle);
} else {
throw new Exception("Unable to open file: {$filePath}");
}
Log::info("Statement Entry data processing completed. Total processed: $processedCount, Total errors: $errorCount");
} catch (Exception $e) {
Log::error('Error in ProcessStmtEntryDataJob: ' . $e->getMessage());
throw $e;

121
app/Jobs/ProcessStmtNarrFormatDataJob.php
View File

@@ -1,69 +1,112 @@
<?php
namespace Modules\Webstatement\Jobs;
namespace Modules\Webstatement\Jobs;
use Exception;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Log;
use Modules\Webstatement\Models\TempStmtNarrFormat;
use Exception;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
use Modules\Webstatement\Models\TempStmtNarrFormat;
class ProcessStmtNarrFormatDataJob implements ShouldQueue
class ProcessStmtNarrFormatDataJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $periods;
/**
* Create a new job instance.
*/
public function __construct(array $periods = [])
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
$this->periods = $periods;
}
/**
* Create a new job instance.
*/
public function __construct()
{
//
}
/**
* Execute the job.
*/
public function handle(): void
{
try {
set_time_limit(24 * 60 * 60);
$disk = Storage::disk('sftpStatement');
$processedCount = 0;
$errorCount = 0;
/**
* Execute the job.
*/
public function handle()
: void
{
$filePath = storage_path('app/20250207.ST.STMT.NARR.FORMAT.csv'); // Adjust this path as needed
try {
if (!file_exists($filePath)) {
throw new Exception("File not found: $filePath");
if (empty($this->periods)) {
Log::warning('No periods provided for statement narrative format data processing');
return;
}
foreach ($this->periods as $period) {
// Skip the _parameter folder
if ($period === '_parameter') {
Log::info("Skipping _parameter folder");
continue;
}
set_time_limit(24 * 60 * 60);
// Construct the filename based on the period folder name
$filename = "$period.ST.STMT.NARR.FORMAT.csv";
$filePath = "$period/$filename";
if (!file_exists($filePath)) {
throw new Exception("File not found: {$filePath}");
Log::info("Processing statement narrative format file: $filePath");
if (!$disk->exists($filePath)) {
Log::warning("File not found: $filePath");
continue;
}
$handle = fopen($filePath, "r");
// Create a temporary local copy of the file
$tempFilePath = storage_path("app/temp_$filename");
file_put_contents($tempFilePath, $disk->get($filePath));
$handle = fopen($tempFilePath, "r");
if ($handle !== false) {
$headers = (new TempStmtNarrFormat())->getFillable();
$rowCount = 0;
while (($row = fgetcsv($handle, 0, "~")) !== false) {
$rowCount++;
if (count($headers) === count($row)) {
$data = array_combine($headers, $row);
try {
if($data['_id'] !== 'id') {
TempStmtNarrFormat::updateOrCreate(['_id' => $data['_id']], $data);
if (isset($data['_id']) && $data['_id'] !== 'id' && $data['_id'] !== '_id') {
TempStmtNarrFormat::updateOrCreate(
['_id' => $data['_id']],
$data
);
$processedCount++;
}
} catch (Exception $e) {
Log::error('Error processing stmt narr format: ' . $e->getMessage());
$errorCount++;
Log::error("Error processing Statement Narrative Format at row $rowCount in $filePath: " . $e->getMessage());
}
} else {
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row));
}
}
fclose($handle);
Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors.");
// Clean up the temporary file
unlink($tempFilePath);
} else {
throw new Exception("Unable to open file: {$filePath}");
Log::error("Unable to open file: $filePath");
}
} catch (Exception $e) {
Log::error('Error in ProcessStmtNarrFormatDataJob: ' . $e->getMessage());
throw $e;
}
Log::info("Statement Narrative Format data processing completed. Total processed: $processedCount, Total errors: $errorCount");
} catch (Exception $e) {
Log::error('Error in ProcessStmtNarrFormatDataJob: ' . $e->getMessage());
throw $e;
}
}
}

94
app/Jobs/ProcessStmtNarrParamDataJob.php
View File

@@ -8,60 +8,102 @@
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Log;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
use Modules\Webstatement\Models\TempStmtNarrParam;
class ProcessStmtNarrParamDataJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $periods;
/**
* Create a new job instance.
*/
public function __construct()
public function __construct(array $periods = [])
{
//
$this->periods = $periods;
}
/**
* Execute the job.
*/
public function handle()
: void
public function handle(): void
{
$filePath = storage_path('app/20250207.ST.STMT.NARR.PARAM.csv'); // Adjust this path as needed
try {
if (!file_exists($filePath)) {
throw new Exception("File not found: $filePath");
}
set_time_limit(24 * 60 * 60);
$disk = Storage::disk('sftpStatement');
$processedCount = 0;
$errorCount = 0;
if (!file_exists($filePath)) {
throw new Exception("File not found: {$filePath}");
if (empty($this->periods)) {
Log::warning('No periods provided for statement narrative parameter data processing');
return;
}
$handle = fopen($filePath, "r");
foreach ($this->periods as $period) {
// Skip the _parameter folder
if ($period === '_parameter') {
Log::info("Skipping _parameter folder");
continue;
}
if ($handle !== false) {
$headers = (new TempStmtNarrParam())->getFillable();
while (($row = fgetcsv($handle, 0, "~")) !== false) {
if (count($headers) === count($row)) {
$data = array_combine($headers, $row);
// Construct the filename based on the period folder name
$filename = "$period.ST.STMT.NARR.PARAM.csv";
$filePath = "$period/$filename";
try {
if($data['_id'] !== 'id'){
TempStmtNarrParam::updateOrCreate(['_id' => $data['_id']], $data);
Log::info("Processing statement narrative parameter file: $filePath");
if (!$disk->exists($filePath)) {
Log::warning("File not found: $filePath");
continue;
}
// Create a temporary local copy of the file
$tempFilePath = storage_path("app/temp_$filename");
file_put_contents($tempFilePath, $disk->get($filePath));
$handle = fopen($tempFilePath, "r");
if ($handle !== false) {
$headers = (new TempStmtNarrParam())->getFillable();
$rowCount = 0;
while (($row = fgetcsv($handle, 0, "~")) !== false) {
$rowCount++;
if (count($headers) === count($row)) {
$data = array_combine($headers, $row);
try {
if (isset($data['_id']) && $data['_id'] !== 'id' && $data['_id'] !== '_id') {
TempStmtNarrParam::updateOrCreate(
['_id' => $data['_id']],
$data
);
$processedCount++;
}
} catch (Exception $e) {
$errorCount++;
Log::error("Error processing Statement Narrative Parameter at row $rowCount in $filePath: " . $e->getMessage());
}
} catch (Exception $e) {
Log::error('Error processing stmt narr param: ' . $e->getMessage());
} else {
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row));
}
}
fclose($handle);
Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors.");
// Clean up the temporary file
unlink($tempFilePath);
} else {
Log::error("Unable to open file: $filePath");
}
fclose($handle);
} else {
throw new Exception("Unable to open file: {$filePath}");
}
Log::info("Statement Narrative Parameter data processing completed. Total processed: $processedCount, Total errors: $errorCount");
} catch (Exception $e) {
Log::error('Error in ProcessStmtNarrParamDataJob: ' . $e->getMessage());
throw $e;

122
app/Jobs/ProcessTransactionDataJob.php
View File

@@ -1,70 +1,112 @@
<?php
namespace Modules\Webstatement\Jobs;
namespace Modules\Webstatement\Jobs;
use Exception;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
use Modules\Webstatement\Models\TempTransaction;
use Exception;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
use Modules\Webstatement\Models\TempTransaction;
class ProcessTransactionDataJob implements ShouldQueue
class ProcessTransactionDataJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $periods;
/**
* Create a new job instance.
*/
public function __construct(array $periods = [])
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
$this->periods = $periods;
}
/**
* Create a new job instance.
*/
public function __construct()
{
//
}
/**
* Execute the job.
*/
public function handle(): void
{
try {
set_time_limit(24 * 60 * 60);
$disk = Storage::disk('sftpStatement');
$processedCount = 0;
$errorCount = 0;
/**
* Execute the job.
*/
public function handle()
: void
{
$filePath = storage_path('app/20250207.ST.TRANSACTION.csv'); // Adjust this path as needed
try {
if (!file_exists($filePath)) {
throw new Exception("File not found: $filePath");
if (empty($this->periods)) {
Log::warning('No periods provided for transaction data processing');
return;
}
foreach ($this->periods as $period) {
// Skip the _parameter folder
if ($period === '_parameter') {
Log::info("Skipping _parameter folder");
continue;
}
set_time_limit(24 * 60 * 60);
// Construct the filename based on the period folder name
$filename = "$period.ST.TRANSACTION.csv";
$filePath = "$period/$filename";
if (!file_exists($filePath)) {
throw new Exception("File not found: {$filePath}");
Log::info("Processing transaction file: $filePath");
if (!$disk->exists($filePath)) {
Log::warning("File not found: $filePath");
continue;
}
$handle = fopen($filePath, "r");
// Create a temporary local copy of the file
$tempFilePath = storage_path("app/temp_$filename");
file_put_contents($tempFilePath, $disk->get($filePath));
$handle = fopen($tempFilePath, "r");
if ($handle !== false) {
$headers = (new TempTransaction())->getFillable();
$rowCount = 0;
while (($row = fgetcsv($handle, 0, "~")) !== false) {
$rowCount++;
if (count($headers) === count($row)) {
$data = array_combine($headers, $row);
try {
if($data['_id'] !== 'id') {
TempTransaction::updateOrCreate(['_id' => $data['_id']], $data);
if (isset($data['_id']) && $data['_id'] !== 'id' && $data['_id'] !== '_id') {
TempTransaction::updateOrCreate(
['_id' => $data['_id']],
$data
);
$processedCount++;
}
} catch (Exception $e) {
Log::error('Error processing transactions: ' . $e->getMessage());
$errorCount++;
Log::error("Error processing Transaction at row $rowCount in $filePath: " . $e->getMessage());
}
} else {
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row));
}
}
fclose($handle);
Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors.");
// Clean up the temporary file
unlink($tempFilePath);
} else {
throw new Exception("Unable to open file: {$filePath}");
Log::error("Unable to open file: $filePath");
}
} catch (Exception $e) {
Log::error('Error in ProcessTransctionDataJob: ' . $e->getMessage());
throw $e;
}
Log::info("Transaction data processing completed. Total processed: $processedCount, Total errors: $errorCount");
} catch (Exception $e) {
Log::error('Error in ProcessTransactionDataJob: ' . $e->getMessage());
throw $e;
}
}
}

29
database/migrations/2025_05_20_083916_fix_accounts_table.php Normal file
View File

@@ -0,0 +1,29 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('accounts',function(Blueprint $table){
$table->string('open_actual_bal')->nullable();
$table->string('open_cleared_bal')->nullable();
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('accounts', function (Blueprint $table){
$table->dropColumn(['open_cleared_bal','open_actual_bal']);
});
}
};