✨ feat(customer): tambah field alamat lengkap & pemrosesan CSV dinamis
- Tambah 13 field alamat KTP & domisili di tabel customers (nullable, aman untuk rollback) - Update model Customer: fillable & casting tanggal - ProcessCustomerDataJob: header CSV dinamis, mapping otomatis, trim value - Batch save pakai DB::transaction(), logging detail, error handling lengkap - Fleksibel untuk CSV dengan header bervariasi & backward-compatible
This commit is contained in:
Daeng Deni Mardaeni
@@ -8,6 +8,7 @@
|
||||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Illuminate\Support\Facades\Storage;
|
||||
use Modules\Webstatement\Models\Customer;
|
||||
@@ -112,13 +113,28 @@
|
||||
return;
|
||||
}
|
||||
|
||||
$headers = (new Customer())->getFillable();
|
||||
// Read header from CSV file
|
||||
$csvHeaders = fgetcsv($handle, 0, self::CSV_DELIMITER);
|
||||
if ($csvHeaders === false) {
|
||||
Log::error("Unable to read headers from file: $filePath");
|
||||
fclose($handle);
|
||||
return;
|
||||
}
|
||||
|
||||
// Map CSV headers to database fields
|
||||
$headerMapping = $this->getHeaderMapping($csvHeaders);
|
||||
|
||||
Log::info("CSV Headers found", [
|
||||
'csv_headers' => $csvHeaders,
|
||||
'mapped_fields' => array_values($headerMapping)
|
||||
]);
|
||||
|
||||
$rowCount = 0;
|
||||
$chunkCount = 0;
|
||||
|
||||
while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) {
|
||||
$rowCount++;
|
||||
$this->processRow($row, $headers, $rowCount, $filePath);
|
||||
$this->processRow($row, $csvHeaders, $headerMapping, $rowCount, $filePath);
|
||||
|
||||
// Process in chunks to avoid memory issues
|
||||
if (count($this->customerBatch) >= self::CHUNK_SIZE) {
|
||||
@@ -137,16 +153,61 @@
|
||||
Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors.");
|
||||
}
|
||||
|
||||
private function processRow(array $row, array $headers, int $rowCount, string $filePath)
|
||||
/**
|
||||
* Map CSV headers to database field names
|
||||
* Memetakan header CSV ke nama field database
|
||||
*/
|
||||
private function getHeaderMapping(array $csvHeaders): array
|
||||
{
|
||||
$mapping = [];
|
||||
$fillableFields = (new Customer())->getFillable();
|
||||
|
||||
foreach ($csvHeaders as $index => $csvHeader) {
|
||||
$csvHeader = trim($csvHeader);
|
||||
|
||||
// Direct mapping untuk field yang sama
|
||||
if (in_array($csvHeader, $fillableFields)) {
|
||||
$mapping[$index] = $csvHeader;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Custom mapping untuk field yang berbeda nama
|
||||
$customMapping = [
|
||||
'co_code' => 'branch_code', // co_code di CSV menjadi branch_code di database
|
||||
];
|
||||
|
||||
if (isset($customMapping[$csvHeader])) {
|
||||
$mapping[$index] = $customMapping[$csvHeader];
|
||||
} else {
|
||||
// Jika field ada di fillable, gunakan langsung
|
||||
if (in_array($csvHeader, $fillableFields)) {
|
||||
$mapping[$index] = $csvHeader;
|
||||
}
|
||||
// Jika tidak ada mapping, skip field ini
|
||||
}
|
||||
}
|
||||
|
||||
return $mapping;
|
||||
}
|
||||
|
||||
private function processRow(array $row, array $csvHeaders, array $headerMapping, int $rowCount, string $filePath)
|
||||
: void
|
||||
{
|
||||
if (count($headers) !== count($row)) {
|
||||
if (count($csvHeaders) !== count($row)) {
|
||||
Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " .
|
||||
count($headers) . ", Got: " . count($row));
|
||||
count($csvHeaders) . ", Got: " . count($row));
|
||||
return;
|
||||
}
|
||||
|
||||
$data = array_combine($headers, $row);
|
||||
// Map CSV data to database fields
|
||||
$data = [];
|
||||
foreach ($row as $index => $value) {
|
||||
if (isset($headerMapping[$index])) {
|
||||
$fieldName = $headerMapping[$index];
|
||||
$data[$fieldName] = trim($value);
|
||||
}
|
||||
}
|
||||
|
||||
$this->addToBatch($data, $rowCount, $filePath);
|
||||
}
|
||||
|
||||
@@ -175,12 +236,20 @@
|
||||
|
||||
/**
|
||||
* Save batched records to the database
|
||||
* Menyimpan data customer dalam batch ke database dengan transaksi
|
||||
*/
|
||||
private function saveBatch()
|
||||
: void
|
||||
{
|
||||
if (empty($this->customerBatch)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$batchSize = count($this->customerBatch);
|
||||
Log::info("Starting batch save", ['batch_size' => $batchSize]);
|
||||
|
||||
try {
|
||||
if (!empty($this->customerBatch)) {
|
||||
DB::transaction(function () use ($batchSize) {
|
||||
// Bulk insert/update customers
|
||||
Customer::upsert(
|
||||
$this->customerBatch,
|
||||
@@ -188,14 +257,26 @@
|
||||
array_diff((new Customer())->getFillable(), ['customer_code']) // Update columns
|
||||
);
|
||||
|
||||
// Reset customer batch after processing
|
||||
$this->customerBatch = [];
|
||||
}
|
||||
Log::info("Batch save completed successfully", ['batch_size' => $batchSize]);
|
||||
});
|
||||
|
||||
// Reset customer batch after successful processing
|
||||
$this->customerBatch = [];
|
||||
|
||||
} catch (Exception $e) {
|
||||
Log::error("Error in saveBatch: " . $e->getMessage());
|
||||
$this->errorCount += count($this->customerBatch);
|
||||
Log::error("Error in saveBatch", [
|
||||
'error' => $e->getMessage(),
|
||||
'batch_size' => $batchSize,
|
||||
'trace' => $e->getTraceAsString()
|
||||
]);
|
||||
|
||||
$this->errorCount += $batchSize;
|
||||
|
||||
// Reset batch even if there's an error to prevent reprocessing the same failed records
|
||||
$this->customerBatch = [];
|
||||
|
||||
// Re-throw exception untuk handling di level atas
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -29,8 +29,33 @@ class Customer extends Model
|
||||
'home_rw',
|
||||
'ktp_rt',
|
||||
'ktp_rw',
|
||||
'local_ref'
|
||||
'local_ref',
|
||||
'ktp_kelurahan',
|
||||
'ktp_kecamatan',
|
||||
'town_country',
|
||||
'ktp_provinsi',
|
||||
'post_code',
|
||||
'l_dom_street',
|
||||
'l_dom_rt',
|
||||
'l_dom_kelurahan',
|
||||
'l_dom_rw',
|
||||
'l_dom_kecamatan',
|
||||
'l_dom_provinsi',
|
||||
'l_dom_t_country',
|
||||
'l_dom_post_code'
|
||||
];
|
||||
|
||||
/**
|
||||
* Get the attributes that should be cast.
|
||||
* Mendefinisikan casting untuk field-field tertentu
|
||||
*/
|
||||
protected function casts(): array
|
||||
{
|
||||
return [
|
||||
'date_of_birth' => 'date',
|
||||
'birth_incorp_date' => 'date',
|
||||
];
|
||||
}
|
||||
public function accounts(){
|
||||
return $this->hasMany(Account::class, 'customer_code', 'customer_code');
|
||||
}
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
<?php
|
||||
|
||||
use Illuminate\Database\Migrations\Migration;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
use Illuminate\Support\Facades\Schema;
|
||||
|
||||
return new class extends Migration
|
||||
{
|
||||
/**
|
||||
* Run the migrations.
|
||||
* Menambahkan field-field yang belum ada pada tabel customers
|
||||
*/
|
||||
public function up(): void
|
||||
{
|
||||
Schema::table('customers', function (Blueprint $table) {
|
||||
// Field yang belum ada berdasarkan CSV header
|
||||
$table->string('ktp_kelurahan')->nullable()->after('local_ref')->comment('Kelurahan sesuai KTP');
|
||||
$table->string('ktp_kecamatan')->nullable()->after('ktp_kelurahan')->comment('Kecamatan sesuai KTP');
|
||||
$table->string('town_country')->nullable()->after('ktp_kecamatan')->comment('Kota/Negara');
|
||||
$table->string('ktp_provinsi')->nullable()->after('town_country')->comment('Provinsi sesuai KTP');
|
||||
$table->string('post_code')->nullable()->after('ktp_provinsi')->comment('Kode pos alternatif');
|
||||
$table->string('l_dom_street')->nullable()->after('post_code')->comment('Alamat domisili - jalan');
|
||||
$table->string('l_dom_rt')->nullable()->after('l_dom_street')->comment('Alamat domisili - RT');
|
||||
$table->string('l_dom_kelurahan')->nullable()->after('l_dom_rt')->comment('Alamat domisili - kelurahan');
|
||||
$table->string('l_dom_rw')->nullable()->after('l_dom_kelurahan')->comment('Alamat domisili - RW');
|
||||
$table->string('l_dom_kecamatan')->nullable()->after('l_dom_rw')->comment('Alamat domisili - kecamatan');
|
||||
$table->string('l_dom_provinsi')->nullable()->after('l_dom_kecamatan')->comment('Alamat domisili - provinsi');
|
||||
$table->string('l_dom_t_country')->nullable()->after('l_dom_provinsi')->comment('Alamat domisili - kota/negara');
|
||||
$table->string('l_dom_post_code')->nullable()->after('l_dom_t_country')->comment('Alamat domisili - kode pos');
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Reverse the migrations.
|
||||
* Menghapus field-field yang ditambahkan
|
||||
*/
|
||||
public function down(): void
|
||||
{
|
||||
Schema::table('customers', function (Blueprint $table) {
|
||||
$table->dropColumn([
|
||||
'ktp_kelurahan',
|
||||
'ktp_kecamatan',
|
||||
'town_country',
|
||||
'ktp_provinsi',
|
||||
'post_code',
|
||||
'l_dom_street',
|
||||
'l_dom_rt',
|
||||
'l_dom_kelurahan',
|
||||
'l_dom_rw',
|
||||
'l_dom_kecamatan',
|
||||
'l_dom_provinsi',
|
||||
'l_dom_t_country',
|
||||
'l_dom_post_code'
|
||||
]);
|
||||
});
|
||||
}
|
||||
};
|
||||
Reference in New Issue
Block a user