From 285c2409ea08d95d50b2189d70a2567366f2f7a5 Mon Sep 17 00:00:00 2001 From: Daeng Deni Mardaeni Date: Wed, 21 May 2025 21:36:45 +0700 Subject: [PATCH] feat(webstatement): tambah fitur pemrosesan data kategori - Menambahkan `ProcessCategoryDataJob` untuk memproses data kategori dari file CSV yang diambil melalui SFTP. - Membuat model `Category` dengan atribut-atribut: - `id_category` - `date_time` - `description` - `short_name` - `system_ind` - `record_status` - `co_code` - `curr_no` - `l_db_cr_ind` - `category_code` - Menambahkan endpoint baru `ProcessCategoryData` di `MigrasiController` untuk memanggil job pemrosesan data kategori. - Menambahkan migrasi untuk membuat tabel `categories` dengan kolom-kolom yang relevan. - Memperbaiki bug pada `ProcessStmtEntryDataJob` dengan menambahkan validasi khusus untuk menghindari pemrosesan baris header yang tidak valid. - Menghapus pemanggilan job yang tidak diperlukan di `MigrasiController`. - Mengupdate logika pemrosesan file untuk memastikan integritas data dalam job kategori. Signed-off-by: Daeng Deni Mardaeni --- app/Http/Controllers/MigrasiController.php | 36 +++-- app/Jobs/ProcessCategoryDataJob.php | 138 ++++++++++++++++++ app/Jobs/ProcessStmtEntryDataJob.php | 12 +- app/Models/Category.php | 24 +++ ...5_05_21_142915_create_categories_table.php | 37 +++++ 5 files changed, 230 insertions(+), 17 deletions(-) create mode 100644 app/Jobs/ProcessCategoryDataJob.php create mode 100644 app/Models/Category.php create mode 100644 database/migrations/2025_05_21_142915_create_categories_table.php diff --git a/app/Http/Controllers/MigrasiController.php b/app/Http/Controllers/MigrasiController.php index 9cdc0ae..b11e030 100644 --- a/app/Http/Controllers/MigrasiController.php +++ b/app/Http/Controllers/MigrasiController.php @@ -9,6 +9,7 @@ use Log; use Modules\Webstatement\Jobs\ProcessAccountDataJob; use Modules\Webstatement\Jobs\ProcessArrangementDataJob; use Modules\Webstatement\Jobs\ProcessBillDetailDataJob; +use Modules\Webstatement\Jobs\ProcessCategoryDataJob; use Modules\Webstatement\Jobs\ProcessCompanyDataJob; use Modules\Webstatement\Jobs\ProcessCustomerDataJob; use Modules\Webstatement\Jobs\ProcessDataCaptureDataJob; @@ -133,7 +134,6 @@ class MigrasiController extends Controller } } - public function ProcessDataCaptureData($periods){ try { ProcessDataCaptureDataJob::dispatch($periods); @@ -143,6 +143,15 @@ class MigrasiController extends Controller } } + public function ProcessCategoryData($periods){ + try { + ProcessCategoryDataJob::dispatch($periods); + return response()->json(['message' => 'Data TempStmtEntry processing job has been successfully']); + } catch (Exception $e) { + return response()->json(['error' => $e->getMessage()], 500); + } + } + public function index() { @@ -152,10 +161,9 @@ class MigrasiController extends Controller $allDirectories = $disk->directories(); //$this->processTransactionData(['_parameter']); - $this->processStmtNarrParamData(['_parameter']); + //$this->processStmtNarrParamData(['_parameter']); //$this->processStmtNarrFormatData(['_parameter']); - - $this->ProcessFtTxnTypeConditioData(['_parameter']); + //$this->ProcessFtTxnTypeConditioData(['_parameter']); // Filter out the _parameter folder $periods = array_filter($allDirectories, function($dir) { @@ -171,15 +179,19 @@ class MigrasiController extends Controller return response()->json(['message' => 'No valid period folders found in SFTP storage'], 404); } - $this->processCustomerData(['20250519']); - //$this->processAccountData(['20250519']); - $this->processArrangementData(['20250512']); - $this->processBillDetailData(['20250512']); - $this->processFundsTransferData(['20250512']); - $this->processStmtEntryData(['20250512']); + $this->ProcessCategoryData($periods); + + //$this->processCustomerData($periods); + //$this->processAccountData($periods); + + //$this->processStmtEntryData($periods); + //$this->ProcessDataCaptureData($periods); + //$this->processFundsTransferData($periods); + + //$this->processArrangementData($periods); + //$this->processBillDetailData($periods); + - //$this->ProcessCompanyData($periods); - $this->ProcessDataCaptureData($periods); return response()->json(['message' => 'Data processing job has been successfully']); } diff --git a/app/Jobs/ProcessCategoryDataJob.php b/app/Jobs/ProcessCategoryDataJob.php new file mode 100644 index 0000000..0fffdd0 --- /dev/null +++ b/app/Jobs/ProcessCategoryDataJob.php @@ -0,0 +1,138 @@ +periods = $periods; + } + + /** + * Execute the job. + */ + public function handle() + : void + { + try { + set_time_limit(24 * 60 * 60); + $disk = Storage::disk('sftpStatement'); + $processedCount = 0; + $errorCount = 0; + + if (empty($this->periods)) { + Log::warning('No periods provided for category data processing'); + return; + } + + foreach ($this->periods as $period) { + // Skip the _parameter folder + if ($period === '_parameter') { + Log::info("Skipping _parameter folder"); + continue; + } + + // Construct the filename based on the period folder name + $filename = "$period.ST.CATEGORY.csv"; + $filePath = "$period/$filename"; + + Log::info("Processing category file: $filePath"); + + if (!$disk->exists($filePath)) { + Log::warning("File not found: $filePath"); + continue; + } + + // Create a temporary local copy of the file + $tempFilePath = storage_path("app/temp_$filename"); + file_put_contents($tempFilePath, $disk->get($filePath)); + + $handle = fopen($tempFilePath, "r"); + + if ($handle !== false) { + // Get the headers from the first row + $headerRow = fgetcsv($handle, 0, "~"); + + // Map the headers to our model fields + $headerMap = [ + 'id' => 'id_category', + 'date_time' => 'date_time', + 'description' => 'description', + 'short_name' => 'short_name', + 'system_ind' => 'system_ind', + 'record_status' => 'record_status', + 'co_code' => 'co_code', + 'curr_no' => 'curr_no', + 'l_db_cr_ind' => 'l_db_cr_ind', + 'category_code' => 'category_code' + ]; + + $rowCount = 0; + + while (($row = fgetcsv($handle, 0, "~")) !== false) { + $rowCount++; + + if (count($headerRow) === count($row)) { + // Combine the header row with the data row + $rawData = array_combine($headerRow, $row); + + // Map the raw data to our model fields + $data = []; + foreach ($headerMap as $csvField => $modelField) { + $data[$modelField] = $rawData[$csvField] ?? null; + } + + try { + // Skip header row if it was included in the data + if ($data['id_category'] !== 'id') { + // Use firstOrNew instead of updateOrCreate + $category = Category::firstOrNew(['id_category' => $data['id_category']]); + $category->fill($data); + $category->save(); + $processedCount++; + } + } catch (Exception $e) { + $errorCount++; + Log::error("Error processing Category at row $rowCount in $filePath: " . $e->getMessage()); + } + } else { + Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($headerRow) . ", Got: " . count($row)); + } + } + + fclose($handle); + Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors."); + + // Clean up the temporary file + unlink($tempFilePath); + } else { + Log::error("Unable to open file: $filePath"); + } + } + + Log::info("Category data processing completed. Total processed: $processedCount, Total errors: $errorCount"); + + } catch (Exception $e) { + Log::error('Error in ProcessCategoryDataJob: ' . $e->getMessage()); + throw $e; + } + } + } diff --git a/app/Jobs/ProcessStmtEntryDataJob.php b/app/Jobs/ProcessStmtEntryDataJob.php index 3739204..82e7581 100644 --- a/app/Jobs/ProcessStmtEntryDataJob.php +++ b/app/Jobs/ProcessStmtEntryDataJob.php @@ -77,11 +77,13 @@ if (count($headers) === count($row)) { $data = array_combine($headers, $row); try { - StmtEntry::updateOrCreate( - ['stmt_entry_id' => $data['stmt_entry_id']], - $data - ); - $processedCount++; + if ($data['stmt_entry_id'] !== 'stmt_entry_id') { + StmtEntry::updateOrCreate( + ['stmt_entry_id' => $data['stmt_entry_id']], + $data + ); + $processedCount++; + } } catch (Exception $e) { $errorCount++; Log::error("Error processing Statement Entry at row $rowCount in $filePath: " . $e->getMessage()); diff --git a/app/Models/Category.php b/app/Models/Category.php new file mode 100644 index 0000000..786dc8a --- /dev/null +++ b/app/Models/Category.php @@ -0,0 +1,24 @@ +id(); + $table->string('id_category')->nullable(); + $table->string('date_time')->nullable(); + $table->text('description')->nullable(); + $table->string('short_name')->nullable(); + $table->string('system_ind')->nullable(); + $table->string('record_status')->nullable(); + $table->string('co_code')->nullable(); + $table->string('curr_no')->nullable(); + $table->string('l_db_cr_ind')->nullable(); + $table->string('category_code')->nullable(); + $table->timestamps(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::dropIfExists('categories'); + } + };