feat(webstatement): tambah fitur pemrosesan data "Data Capture"
- Menambahkan fungsi baru `ProcessDataCaptureData` di `MigrasiController` untuk menjadwalkan pemrosesan data "Data Capture". - Menambahkan job baru `ProcessDataCaptureDataJob`: - Mengambil data "Data Capture" dari file CSV melalui SFTP. - Memproses dan menyimpan data ke database dengan validasi dan logging. - Mendukung operasi pembaruan (update) atau pembuatan (insert) data menggunakan Eloquent `updateOrCreate`. - Menambahkan model baru `DataCapture`: - Memetakan data ke tabel `data_captures` di database. - Mendukung properti `fillable` dan `casts` untuk format data yang valid, termasuk konversi nilai decimal dan tanggal. - Menambahkan migrasi baru `create_data_captures_table` untuk membuat tabel `data_captures`: - Mendefinisikan semua kolom sesuai kebutuhan data "Data Capture". - Menetapkan tipe data yang relevan seperti decimal, date, dan datetime. - Memperbarui metode `migrasiAll` di `MigrasiController` untuk memanggil fungsi pemrosesan baru `ProcessDataCaptureData`. Signed-off-by: Daeng Deni Mardaeni <ddeni05@gmail.com>
This commit is contained in:
Daeng Deni Mardaeni
@@ -11,6 +11,7 @@ use Modules\Webstatement\Jobs\ProcessArrangementDataJob;
|
||||
use Modules\Webstatement\Jobs\ProcessBillDetailDataJob;
|
||||
use Modules\Webstatement\Jobs\ProcessCompanyDataJob;
|
||||
use Modules\Webstatement\Jobs\ProcessCustomerDataJob;
|
||||
use Modules\Webstatement\Jobs\ProcessDataCaptureDataJob;
|
||||
use Modules\Webstatement\Jobs\ProcessFtTxnTypeConditionJob;
|
||||
use Modules\Webstatement\Jobs\ProcessFundsTransferDataJob;
|
||||
use Modules\Webstatement\Jobs\ProcessStmtEntryDataJob;
|
||||
@@ -133,6 +134,16 @@ class MigrasiController extends Controller
|
||||
}
|
||||
|
||||
|
||||
public function ProcessDataCaptureData($periods){
|
||||
try {
|
||||
ProcessDataCaptureDataJob::dispatch($periods);
|
||||
return response()->json(['message' => 'Data TempStmtEntry processing job has been successfully']);
|
||||
} catch (Exception $e) {
|
||||
return response()->json(['error' => $e->getMessage()], 500);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public function index()
|
||||
{
|
||||
$disk = Storage::disk('sftpStatement');
|
||||
@@ -168,6 +179,7 @@ class MigrasiController extends Controller
|
||||
$this->processStmtEntryData($periods);
|
||||
|
||||
$this->ProcessCompanyData($periods);
|
||||
$this->ProcessDataCaptureData($periods);
|
||||
|
||||
return response()->json(['message' => 'Data processing job has been successfully']);
|
||||
}
|
||||
|
||||
185
app/Jobs/ProcessDataCaptureDataJob.php
Normal file
185
app/Jobs/ProcessDataCaptureDataJob.php
Normal file
@@ -0,0 +1,185 @@
|
||||
<?php
|
||||
|
||||
namespace Modules\Webstatement\Jobs;
|
||||
|
||||
use Exception;
|
||||
use Illuminate\Bus\Queueable;
|
||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use Modules\Webstatement\Models\DataCapture;
|
||||
|
||||
class ProcessDataCaptureDataJob implements ShouldQueue
|
||||
{
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
protected $periods;
|
||||
protected $filename;
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
*/
|
||||
public function __construct(array $periods = [], string $filename = "ST.DATA.CAPTURE.csv")
|
||||
{
|
||||
$this->periods = $periods;
|
||||
$this->filename = $filename;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the job.
|
||||
*/
|
||||
public function handle()
|
||||
: void
|
||||
{
|
||||
try {
|
||||
set_time_limit(24 * 60 * 60);
|
||||
$disk = Storage::disk('sftpStatement');
|
||||
$processedCount = 0;
|
||||
$errorCount = 0;
|
||||
|
||||
if (empty($this->periods)) {
|
||||
Log::warning('No periods provided for data capture processing');
|
||||
return;
|
||||
}
|
||||
|
||||
foreach ($this->periods as $period) {
|
||||
// Skip the _parameter folder
|
||||
if ($period === '_parameter') {
|
||||
Log::info("Skipping _parameter folder");
|
||||
continue;
|
||||
}
|
||||
|
||||
// Construct the filepath based on the period folder name
|
||||
$fileName = "$period.$this->filename";
|
||||
$filePath = "$period/$fileName";
|
||||
|
||||
Log::info("Processing data capture file: $filePath");
|
||||
|
||||
if (!$disk->exists($filePath)) {
|
||||
Log::warning("File not found: $filePath");
|
||||
continue;
|
||||
}
|
||||
|
||||
// Create a temporary local copy of the file
|
||||
$tempFilePath = storage_path("app/temp_{$this->filename}");
|
||||
file_put_contents($tempFilePath, $disk->get($filePath));
|
||||
|
||||
$handle = fopen($tempFilePath, "r");
|
||||
|
||||
if ($handle !== false) {
|
||||
// CSV headers from the file
|
||||
$csvHeaders = [
|
||||
'id',
|
||||
'account_number',
|
||||
'sign',
|
||||
'amount_lcy',
|
||||
'transaction_code',
|
||||
'their_reference',
|
||||
'narrative',
|
||||
'pl_category',
|
||||
'customer_id',
|
||||
'account_officer',
|
||||
'product_category',
|
||||
'value_date',
|
||||
'currency',
|
||||
'amount_fcy',
|
||||
'exchange_rate',
|
||||
'neg_ref_no',
|
||||
'position_type',
|
||||
'our_reference',
|
||||
'reversal_marker',
|
||||
'exposure_date',
|
||||
'currency_market',
|
||||
'iblc_country',
|
||||
'last_version',
|
||||
'otor_version',
|
||||
'department_code',
|
||||
'dealer_desk',
|
||||
'bank_sort_cde',
|
||||
'cheque_number',
|
||||
'accounting_date',
|
||||
'contingent_acct',
|
||||
'cheq_type',
|
||||
'tfs_reference',
|
||||
'accounting_company',
|
||||
'stmt_no',
|
||||
'curr_no',
|
||||
'inputter',
|
||||
'authoriser',
|
||||
'co_code',
|
||||
'date_time'
|
||||
];
|
||||
|
||||
$rowCount = 0;
|
||||
|
||||
while (($row = fgetcsv($handle, 0, "~")) !== false) {
|
||||
$rowCount++;
|
||||
|
||||
// Skip header row if it exists
|
||||
if ($rowCount === 1 && strtolower($row[0]) === 'id') {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (count($csvHeaders) === count($row)) {
|
||||
$data = array_combine($csvHeaders, $row);
|
||||
|
||||
try {
|
||||
// Format dates if they exist
|
||||
foreach (['value_date', 'exposure_date', 'accounting_date'] as $dateField) {
|
||||
if (!empty($data[$dateField])) {
|
||||
try {
|
||||
$data[$dateField] = date('Y-m-d', strtotime($data[$dateField]));
|
||||
} catch (Exception $e) {
|
||||
// If date parsing fails, keep the original value
|
||||
Log::warning("Failed to parse date for $dateField: {$data[$dateField]}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Format datetime if it exists
|
||||
if (!empty($data['date_time'])) {
|
||||
try {
|
||||
$data['date_time'] = date('Y-m-d H:i:s', strtotime($data['date_time']));
|
||||
} catch (Exception $e) {
|
||||
// If datetime parsing fails, keep the original value
|
||||
Log::warning("Failed to parse datetime for date_time: {$data['date_time']}");
|
||||
}
|
||||
}
|
||||
|
||||
if (!empty($data['id'])) {
|
||||
DataCapture::updateOrCreate(
|
||||
['id' => $data['id']],
|
||||
$data
|
||||
);
|
||||
$processedCount++;
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
$errorCount++;
|
||||
Log::error("Error processing Data Capture at row $rowCount in $filePath: " . $e->getMessage());
|
||||
}
|
||||
} else {
|
||||
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . count($csvHeaders) . ", Got: " . count($row));
|
||||
}
|
||||
}
|
||||
|
||||
fclose($handle);
|
||||
Log::info("Completed processing $filePath. Processed $processedCount records with $errorCount errors.");
|
||||
|
||||
// Clean up the temporary file
|
||||
unlink($tempFilePath);
|
||||
} else {
|
||||
Log::error("Unable to open file: $filePath");
|
||||
}
|
||||
}
|
||||
|
||||
Log::info("Data capture processing completed. Total processed: $processedCount, Total errors: $errorCount");
|
||||
|
||||
} catch (Exception $e) {
|
||||
Log::error('Error in ProcessDataCaptureDataJob: ' . $e->getMessage());
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
}
|
||||
74
app/Models/DataCapture.php
Normal file
74
app/Models/DataCapture.php
Normal file
@@ -0,0 +1,74 @@
|
||||
<?php
|
||||
|
||||
namespace Modules\Webstatement\Models;
|
||||
|
||||
use Illuminate\Database\Eloquent\Model;
|
||||
|
||||
class DataCapture extends Model
|
||||
{
|
||||
/**
|
||||
* Indicates if the model's ID is auto-incrementing.
|
||||
*
|
||||
* @var bool
|
||||
*/
|
||||
public $incrementing = false;
|
||||
protected $table = 'data_captures';
|
||||
/**
|
||||
* The data type of the auto-incrementing ID.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $keyType = 'string';
|
||||
|
||||
protected $fillable = [
|
||||
'id',
|
||||
'account_number',
|
||||
'sign',
|
||||
'amount_lcy',
|
||||
'transaction_code',
|
||||
'their_reference',
|
||||
'narrative',
|
||||
'pl_category',
|
||||
'customer_id',
|
||||
'account_officer',
|
||||
'product_category',
|
||||
'value_date',
|
||||
'currency',
|
||||
'amount_fcy',
|
||||
'exchange_rate',
|
||||
'neg_ref_no',
|
||||
'position_type',
|
||||
'our_reference',
|
||||
'reversal_marker',
|
||||
'exposure_date',
|
||||
'currency_market',
|
||||
'iblc_country',
|
||||
'last_version',
|
||||
'otor_version',
|
||||
'department_code',
|
||||
'dealer_desk',
|
||||
'bank_sort_cde',
|
||||
'cheque_number',
|
||||
'accounting_date',
|
||||
'contingent_acct',
|
||||
'cheq_type',
|
||||
'tfs_reference',
|
||||
'accounting_company',
|
||||
'stmt_no',
|
||||
'curr_no',
|
||||
'inputter',
|
||||
'authoriser',
|
||||
'co_code',
|
||||
'date_time'
|
||||
];
|
||||
|
||||
protected $casts = [
|
||||
'amount_lcy' => 'decimal:2',
|
||||
'amount_fcy' => 'decimal:2',
|
||||
'exchange_rate' => 'decimal:6',
|
||||
'value_date' => 'date',
|
||||
'exposure_date' => 'date',
|
||||
'accounting_date' => 'date',
|
||||
'date_time' => 'datetime'
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
<?php
|
||||
|
||||
use Illuminate\Database\Migrations\Migration;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
use Illuminate\Support\Facades\Schema;
|
||||
|
||||
return new class extends Migration
|
||||
{
|
||||
/**
|
||||
* Run the migrations.
|
||||
*/
|
||||
public function up(): void
|
||||
{
|
||||
Schema::create('data_captures', function (Blueprint $table) {
|
||||
$table->string('id')->primary();
|
||||
$table->string('account_number')->nullable();
|
||||
$table->string('sign')->nullable();
|
||||
$table->decimal('amount_lcy', 20, 2)->nullable();
|
||||
$table->string('transaction_code')->nullable();
|
||||
$table->string('their_reference')->nullable();
|
||||
$table->text('narrative')->nullable();
|
||||
$table->string('pl_category')->nullable();
|
||||
$table->string('customer_id')->nullable();
|
||||
$table->string('account_officer')->nullable();
|
||||
$table->string('product_category')->nullable();
|
||||
$table->date('value_date')->nullable();
|
||||
$table->string('currency')->nullable();
|
||||
$table->decimal('amount_fcy', 20, 2)->nullable();
|
||||
$table->decimal('exchange_rate', 20, 6)->nullable();
|
||||
$table->string('neg_ref_no')->nullable();
|
||||
$table->string('position_type')->nullable();
|
||||
$table->string('our_reference')->nullable();
|
||||
$table->string('reversal_marker')->nullable();
|
||||
$table->date('exposure_date')->nullable();
|
||||
$table->string('currency_market')->nullable();
|
||||
$table->string('iblc_country')->nullable();
|
||||
$table->string('last_version')->nullable();
|
||||
$table->string('otor_version')->nullable();
|
||||
$table->string('department_code')->nullable();
|
||||
$table->string('dealer_desk')->nullable();
|
||||
$table->string('bank_sort_cde')->nullable();
|
||||
$table->string('cheque_number')->nullable();
|
||||
$table->date('accounting_date')->nullable();
|
||||
$table->string('contingent_acct')->nullable();
|
||||
$table->string('cheq_type')->nullable();
|
||||
$table->string('tfs_reference')->nullable();
|
||||
$table->string('accounting_company')->nullable();
|
||||
$table->string('stmt_no')->nullable();
|
||||
$table->string('curr_no')->nullable();
|
||||
$table->string('inputter')->nullable();
|
||||
$table->string('authoriser')->nullable();
|
||||
$table->string('co_code')->nullable();
|
||||
$table->dateTime('date_time')->nullable();
|
||||
$table->timestamps();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Reverse the migrations.
|
||||
*/
|
||||
public function down(): void
|
||||
{
|
||||
Schema::dropIfExists('data_captures');
|
||||
}
|
||||
};
|
||||
Reference in New Issue
Block a user