diff --git a/app/Http/Controllers/MigrasiController.php b/app/Http/Controllers/MigrasiController.php index 18c089a..c7dba57 100644 --- a/app/Http/Controllers/MigrasiController.php +++ b/app/Http/Controllers/MigrasiController.php @@ -11,6 +11,7 @@ use Modules\Webstatement\Jobs\ProcessArrangementDataJob; use Modules\Webstatement\Jobs\ProcessBillDetailDataJob; use Modules\Webstatement\Jobs\ProcessCompanyDataJob; use Modules\Webstatement\Jobs\ProcessCustomerDataJob; +use Modules\Webstatement\Jobs\ProcessDataCaptureDataJob; use Modules\Webstatement\Jobs\ProcessFtTxnTypeConditionJob; use Modules\Webstatement\Jobs\ProcessFundsTransferDataJob; use Modules\Webstatement\Jobs\ProcessStmtEntryDataJob; @@ -133,6 +134,16 @@ class MigrasiController extends Controller } + public function ProcessDataCaptureData($periods){ + try { + ProcessDataCaptureDataJob::dispatch($periods); + return response()->json(['message' => 'Data TempStmtEntry processing job has been successfully']); + } catch (Exception $e) { + return response()->json(['error' => $e->getMessage()], 500); + } + } + + public function index() { $disk = Storage::disk('sftpStatement'); @@ -168,6 +179,7 @@ class MigrasiController extends Controller $this->processStmtEntryData($periods); $this->ProcessCompanyData($periods); + $this->ProcessDataCaptureData($periods); return response()->json(['message' => 'Data processing job has been successfully']); } diff --git a/app/Jobs/ProcessDataCaptureDataJob.php b/app/Jobs/ProcessDataCaptureDataJob.php new file mode 100644 index 0000000..1c8bf2d --- /dev/null +++ b/app/Jobs/ProcessDataCaptureDataJob.php @@ -0,0 +1,185 @@ +periods = $periods; + $this->filename = $filename; + } + + /** + * Execute the job. + */ + public function handle() + : void + { + try { + set_time_limit(24 * 60 * 60); + $disk = Storage::disk('sftpStatement'); + $processedCount = 0; + $errorCount = 0; + + if (empty($this->periods)) { + Log::warning('No periods provided for data capture processing'); + return; + } + + foreach ($this->periods as $period) { + // Skip the _parameter folder + if ($period === '_parameter') { + Log::info("Skipping _parameter folder"); + continue; + } + + // Construct the filepath based on the period folder name + $fileName = "$period.$this->filename"; + $filePath = "$period/$fileName"; + + Log::info("Processing data capture file: $filePath"); + + if (!$disk->exists($filePath)) { + Log::warning("File not found: $filePath"); + continue; + } + + // Create a temporary local copy of the file + $tempFilePath = storage_path("app/temp_{$this->filename}"); + file_put_contents($tempFilePath, $disk->get($filePath)); + + $handle = fopen($tempFilePath, "r"); + + if ($handle !== false) { + // CSV headers from the file + $csvHeaders = [ + 'id', + 'account_number', + 'sign', + 'amount_lcy', + 'transaction_code', + 'their_reference', + 'narrative', + 'pl_category', + 'customer_id', + 'account_officer', + 'product_category', + 'value_date', + 'currency', + 'amount_fcy', + 'exchange_rate', + 'neg_ref_no', + 'position_type', + 'our_reference', + 'reversal_marker', + 'exposure_date', + 'currency_market', + 'iblc_country', + 'last_version', + 'otor_version', + 'department_code', + 'dealer_desk', + 'bank_sort_cde', + 'cheque_number', + 'accounting_date', + 'contingent_acct', + 'cheq_type', + 'tfs_reference', + 'accounting_company', + 'stmt_no', + 'curr_no', + 'inputter', + 'authoriser', + 'co_code', + 'date_time' + ]; + + $rowCount = 0; + + while (($row = fgetcsv($handle, 0, "~")) !== false) { + $rowCount++; + + // Skip header row if it exists + if ($rowCount === 1 && strtolower($row[0]) === 'id') { + continue; + } + + if (count($csvHeaders) === count($row)) { + $data = array_combine($csvHeaders, $row); + + try { + // Format dates if they exist + foreach (['value_date', 'exposure_date', 'accounting_date'] as $dateField) { + if (!empty($data[$dateField])) { + try { + $data[$dateField] = date('Y-m-d', strtotime($data[$dateField])); + } catch (Exception $e) { + // If date parsing fails, keep the original value + Log::warning("Failed to parse date for $dateField: {$data[$dateField]}"); + } + } + } + + // Format datetime if it exists + if (!empty($data['date_time'])) { + try { + $data['date_time'] = date('Y-m-d H:i:s', strtotime($data['date_time'])); + } catch (Exception $e) { + // If datetime parsing fails, keep the original value + Log::warning("Failed to parse datetime for date_time: {$data['date_time']}"); + } + } + + if (!empty($data['id'])) { + DataCapture::updateOrCreate( + ['id' => $data['id']], + $data + ); + $processedCount++; + } + } catch (Exception $e) { + $errorCount++; + Log::error("Error processing Data Capture at row $rowCount in $filePath: " . $e->getMessage()); + } + } else { + Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($csvHeaders) . ", Got: " . count($row)); + } + } + + fclose($handle); + Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors."); + + // Clean up the temporary file + unlink($tempFilePath); + } else { + Log::error("Unable to open file: $filePath"); + } + } + + Log::info("Data capture processing completed. Total processed: $processedCount, Total errors: $errorCount"); + + } catch (Exception $e) { + Log::error('Error in ProcessDataCaptureDataJob: ' . $e->getMessage()); + throw $e; + } + } + } diff --git a/app/Models/DataCapture.php b/app/Models/DataCapture.php new file mode 100644 index 0000000..654890a --- /dev/null +++ b/app/Models/DataCapture.php @@ -0,0 +1,74 @@ + 'decimal:2', + 'amount_fcy' => 'decimal:2', + 'exchange_rate' => 'decimal:6', + 'value_date' => 'date', + 'exposure_date' => 'date', + 'accounting_date' => 'date', + 'date_time' => 'datetime' + ]; + } diff --git a/database/migrations/2025_05_20_151042_create_data_captures_table.php b/database/migrations/2025_05_20_151042_create_data_captures_table.php new file mode 100644 index 0000000..47a74d7 --- /dev/null +++ b/database/migrations/2025_05_20_151042_create_data_captures_table.php @@ -0,0 +1,65 @@ +string('id')->primary(); + $table->string('account_number')->nullable(); + $table->string('sign')->nullable(); + $table->decimal('amount_lcy', 20, 2)->nullable(); + $table->string('transaction_code')->nullable(); + $table->string('their_reference')->nullable(); + $table->text('narrative')->nullable(); + $table->string('pl_category')->nullable(); + $table->string('customer_id')->nullable(); + $table->string('account_officer')->nullable(); + $table->string('product_category')->nullable(); + $table->date('value_date')->nullable(); + $table->string('currency')->nullable(); + $table->decimal('amount_fcy', 20, 2)->nullable(); + $table->decimal('exchange_rate', 20, 6)->nullable(); + $table->string('neg_ref_no')->nullable(); + $table->string('position_type')->nullable(); + $table->string('our_reference')->nullable(); + $table->string('reversal_marker')->nullable(); + $table->date('exposure_date')->nullable(); + $table->string('currency_market')->nullable(); + $table->string('iblc_country')->nullable(); + $table->string('last_version')->nullable(); + $table->string('otor_version')->nullable(); + $table->string('department_code')->nullable(); + $table->string('dealer_desk')->nullable(); + $table->string('bank_sort_cde')->nullable(); + $table->string('cheque_number')->nullable(); + $table->date('accounting_date')->nullable(); + $table->string('contingent_acct')->nullable(); + $table->string('cheq_type')->nullable(); + $table->string('tfs_reference')->nullable(); + $table->string('accounting_company')->nullable(); + $table->string('stmt_no')->nullable(); + $table->string('curr_no')->nullable(); + $table->string('inputter')->nullable(); + $table->string('authoriser')->nullable(); + $table->string('co_code')->nullable(); + $table->dateTime('date_time')->nullable(); + $table->timestamps(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::dropIfExists('data_captures'); + } +};