daengdeni/webstatement
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(webstatement): optimalkan proses migrasi data SFTP

Browse Source 
- Menambahkan parameter `$periods` untuk mendukung proses dinamis berdasarkan folder periode dalam SFTP.
- Memperkenalkan penggunaan `Storage::disk('sftpStatement')` untuk membaca file dari SFTP.
- Menyortir folder periode secara menurun berdasarkan tanggal.
- Memproses tiap entitas data (Customer, Account, Arrangement, Bill Details, dll.) berdasarkan file dalam folder periode.
- Menambah log untuk mencatat jumlah proses yang berhasil, baris error, dan peringatan tentang ketidaksesuaian jumlah kolom.
- Menyisipkan mekanisme untuk mengabaikan folder `_parameter`.
- Mengubah pendekatan dari membaca file lokal menjadi menggunakan file sementara (temp).
- Memperbaiki bug pada metode yang sebelumnya menggunakan jalur file statis.
- Memastikan penanganan file CSV yang lebih fleksibel, termasuk kasus di mana jumlah kolom dalam baris melebihi ekspektasi.
- Menambah error handling untuk mencatat baris-baris bermasalah dan menghindari interupsi proses.

Signed-off-by: Daeng Deni Mardaeni <ddeni05@gmail.com>
This commit is contained in:
Daeng Deni Mardaeni
2025-05-20 16:44:59 +07:00
parent 7bb320d718
commit c442b27dad
11 changed files with 748 additions and 297 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
app/Jobs/ProcessAccountDataJob.php
View File

@@ -8,72 +8,113 @@
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Log;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
use Modules\Webstatement\Models\Account;
class ProcessAccountDataJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $periods;
/**
* Create a new job instance.
*/
public function __construct()
public function __construct(array $periods = [])
{
//
$this->periods = $periods;
}
/**
* Execute the job.
*/
public function handle()
: void
public function handle(): void
{
$filePath = storage_path('app/20250207.ST.ACCOUNT.csv'); // Use Storage facade
try {
if (!file_exists($filePath)) {
throw new Exception("File not found: {$filePath}");
set_time_limit(24 * 60 * 60);
$disk = Storage::disk('sftpStatement');
$processedCount = 0;
$errorCount = 0;
if (empty($this->periods)) {
Log::warning('No periods provided for account data processing');
return;
}
set_time_limit(24 * 60 * 60);
foreach ($this->periods as $period) {
// Skip the _parameter folder
if ($period === '_parameter') {
Log::info("Skipping _parameter folder");
continue;
}
$handle = fopen($filePath, "r");
if ($handle !== false) {
// Construct the filename based on the period folder name
$filename = "$period.ST.ACCOUNT.csv";
$filePath = "$period/$filename";
// Use the header row to determine the order of columns
$headers = (new Account())->getFillable();
Log::info('Headers: ' . implode(", ", $headers));
$rowCount = 0;
while (($row = fgetcsv($handle, 0, "~")) !== false) {
if (count($headers) === count($row)) {
$data = array_combine($headers, $row);
Log::info("Processing account file: $filePath");
// Check if start_year_bal is empty and set it to 0 if so
if (empty($data['start_year_bal']) || $data['start_year_bal'] ="" || $data['start_year_bal']== null) {
$data['start_year_bal'] = 0;
}
if (!$disk->exists($filePath)) {
Log::warning("File not found: $filePath");
continue;
}
if (empty($data['closure_date']) || $data['closure_date'] ="" || $data['closure_date']== null) {
$data['closure_date'] = null;
}
// Create a temporary local copy of the file
$tempFilePath = storage_path("app/temp_$filename");
file_put_contents($tempFilePath, $disk->get($filePath));
try {
if( $data['account_number'] !== 'account_number') {
// Use firstOrNew instead of updateOrCreate
$account = Account::firstOrNew(['account_number' => $data['account_number']]);
$account->fill($data);
$account->save();
$handle = fopen($tempFilePath, "r");
if ($handle !== false) {
$headers = (new Account())->getFillable();
Log::info('Headers: ' . implode(", ", $headers));
$rowCount = 0;
while (($row = fgetcsv($handle, 0, "~")) !== false) {
$rowCount++;
if (count($headers) === count($row)) {
$data = array_combine($headers, $row);
// Check if start_year_bal is empty and set it to 0 if so
if (empty($data['start_year_bal']) || $data['start_year_bal'] == "" || $data['start_year_bal'] == null) {
$data['start_year_bal'] = 0;
}
} catch (Exception $e) {
Log::error('Error processing Account: ' . $e->getMessage());
if (empty($data['closure_date']) || $data['closure_date'] == "" || $data['closure_date'] == null) {
$data['closure_date'] = null;
}
try {
if ($data['account_number'] !== 'account_number') {
// Use firstOrNew instead of updateOrCreate
$account = Account::firstOrNew(['account_number' => $data['account_number']]);
$account->fill($data);
$account->save();
$processedCount++;
}
} catch (Exception $e) {
$errorCount++;
Log::error("Error processing Account at row $rowCount in $filePath: " . $e->getMessage());
}
} else {
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headers) . ", Got: " . count($row));
}
}
fclose($handle);
Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors.");
// Clean up the temporary file
unlink($tempFilePath);
} else {
Log::error("Unable to open file: $filePath");
}
fclose($handle);
} else {
throw new Exception("Unable to open file: {$filePath}");
}
Log::info("Account data processing completed. Total processed: $processedCount, Total errors: $errorCount");
} catch (Exception $e) {
Log::error('Error in ProcessAccountDataJob: ' . $e->getMessage());
throw $e;