From 7af5bf2fe54114306655615ddb4c4547cc6ea16b Mon Sep 17 00:00:00 2001 From: Daeng Deni Mardaeni Date: Thu, 7 Aug 2025 08:45:50 +0700 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20feat(customer):=20tambah=20field=20?= =?UTF-8?q?alamat=20lengkap=20&=20pemrosesan=20CSV=20dinamis?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Tambah 13 field alamat KTP & domisili di tabel customers (nullable, aman untuk rollback) - Update model Customer: fillable & casting tanggal - ProcessCustomerDataJob: header CSV dinamis, mapping otomatis, trim value - Batch save pakai DB::transaction(), logging detail, error handling lengkap - Fleksibel untuk CSV dengan header bervariasi & backward-compatible --- app/Jobs/ProcessCustomerDataJob.php | 105 ++++++++++++++++-- app/Models/Customer.php | 27 ++++- ..._add_missing_fields_to_customers_table.php | 57 ++++++++++ 3 files changed, 176 insertions(+), 13 deletions(-) create mode 100644 database/migrations/2025_08_06_164839_add_missing_fields_to_customers_table.php diff --git a/app/Jobs/ProcessCustomerDataJob.php b/app/Jobs/ProcessCustomerDataJob.php index 93327f9..1bd4408 100644 --- a/app/Jobs/ProcessCustomerDataJob.php +++ b/app/Jobs/ProcessCustomerDataJob.php @@ -8,6 +8,7 @@ use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; + use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Storage; use Modules\Webstatement\Models\Customer; @@ -112,13 +113,28 @@ return; } - $headers = (new Customer())->getFillable(); + // Read header from CSV file + $csvHeaders = fgetcsv($handle, 0, self::CSV_DELIMITER); + if ($csvHeaders === false) { + Log::error("Unable to read headers from file: $filePath"); + fclose($handle); + return; + } + + // Map CSV headers to database fields + $headerMapping = $this->getHeaderMapping($csvHeaders); + + Log::info("CSV Headers found", [ + 'csv_headers' => $csvHeaders, + 'mapped_fields' => array_values($headerMapping) + ]); + $rowCount = 0; $chunkCount = 0; while (($row = fgetcsv($handle, 0, self::CSV_DELIMITER)) !== false) { $rowCount++; - $this->processRow($row, $headers, $rowCount, $filePath); + $this->processRow($row, $csvHeaders, $headerMapping, $rowCount, $filePath); // Process in chunks to avoid memory issues if (count($this->customerBatch) >= self::CHUNK_SIZE) { @@ -137,16 +153,61 @@ Log::info("Completed processing $filePath. Processed {$this->processedCount} records with {$this->errorCount} errors."); } - private function processRow(array $row, array $headers, int $rowCount, string $filePath) + /** + * Map CSV headers to database field names + * Memetakan header CSV ke nama field database + */ + private function getHeaderMapping(array $csvHeaders): array + { + $mapping = []; + $fillableFields = (new Customer())->getFillable(); + + foreach ($csvHeaders as $index => $csvHeader) { + $csvHeader = trim($csvHeader); + + // Direct mapping untuk field yang sama + if (in_array($csvHeader, $fillableFields)) { + $mapping[$index] = $csvHeader; + continue; + } + + // Custom mapping untuk field yang berbeda nama + $customMapping = [ + 'co_code' => 'branch_code', // co_code di CSV menjadi branch_code di database + ]; + + if (isset($customMapping[$csvHeader])) { + $mapping[$index] = $customMapping[$csvHeader]; + } else { + // Jika field ada di fillable, gunakan langsung + if (in_array($csvHeader, $fillableFields)) { + $mapping[$index] = $csvHeader; + } + // Jika tidak ada mapping, skip field ini + } + } + + return $mapping; + } + + private function processRow(array $row, array $csvHeaders, array $headerMapping, int $rowCount, string $filePath) : void { - if (count($headers) !== count($row)) { + if (count($csvHeaders) !== count($row)) { Log::warning("Row $rowCount in $filePath has incorrect column count. Expected: " . - count($headers) . ", Got: " . count($row)); + count($csvHeaders) . ", Got: " . count($row)); return; } - $data = array_combine($headers, $row); + // Map CSV data to database fields + $data = []; + foreach ($row as $index => $value) { + if (isset($headerMapping[$index])) { + $fieldName = $headerMapping[$index]; + $data[$fieldName] = trim($value); + } + } + $this->addToBatch($data, $rowCount, $filePath); } @@ -175,12 +236,20 @@ /** * Save batched records to the database + * Menyimpan data customer dalam batch ke database dengan transaksi */ private function saveBatch() : void { + if (empty($this->customerBatch)) { + return; + } + + $batchSize = count($this->customerBatch); + Log::info("Starting batch save", ['batch_size' => $batchSize]); + try { - if (!empty($this->customerBatch)) { + DB::transaction(function () use ($batchSize) { // Bulk insert/update customers Customer::upsert( $this->customerBatch, @@ -188,14 +257,26 @@ array_diff((new Customer())->getFillable(), ['customer_code']) // Update columns ); - // Reset customer batch after processing - $this->customerBatch = []; - } + Log::info("Batch save completed successfully", ['batch_size' => $batchSize]); + }); + + // Reset customer batch after successful processing + $this->customerBatch = []; + } catch (Exception $e) { - Log::error("Error in saveBatch: " . $e->getMessage()); - $this->errorCount += count($this->customerBatch); + Log::error("Error in saveBatch", [ + 'error' => $e->getMessage(), + 'batch_size' => $batchSize, + 'trace' => $e->getTraceAsString() + ]); + + $this->errorCount += $batchSize; + // Reset batch even if there's an error to prevent reprocessing the same failed records $this->customerBatch = []; + + // Re-throw exception untuk handling di level atas + throw $e; } } diff --git a/app/Models/Customer.php b/app/Models/Customer.php index a7726d7..c5d3bfa 100644 --- a/app/Models/Customer.php +++ b/app/Models/Customer.php @@ -29,8 +29,33 @@ class Customer extends Model 'home_rw', 'ktp_rt', 'ktp_rw', - 'local_ref' + 'local_ref', + 'ktp_kelurahan', + 'ktp_kecamatan', + 'town_country', + 'ktp_provinsi', + 'post_code', + 'l_dom_street', + 'l_dom_rt', + 'l_dom_kelurahan', + 'l_dom_rw', + 'l_dom_kecamatan', + 'l_dom_provinsi', + 'l_dom_t_country', + 'l_dom_post_code' ]; + + /** + * Get the attributes that should be cast. + * Mendefinisikan casting untuk field-field tertentu + */ + protected function casts(): array + { + return [ + 'date_of_birth' => 'date', + 'birth_incorp_date' => 'date', + ]; + } public function accounts(){ return $this->hasMany(Account::class, 'customer_code', 'customer_code'); } diff --git a/database/migrations/2025_08_06_164839_add_missing_fields_to_customers_table.php b/database/migrations/2025_08_06_164839_add_missing_fields_to_customers_table.php new file mode 100644 index 0000000..dd743ac --- /dev/null +++ b/database/migrations/2025_08_06_164839_add_missing_fields_to_customers_table.php @@ -0,0 +1,57 @@ +string('ktp_kelurahan')->nullable()->after('local_ref')->comment('Kelurahan sesuai KTP'); + $table->string('ktp_kecamatan')->nullable()->after('ktp_kelurahan')->comment('Kecamatan sesuai KTP'); + $table->string('town_country')->nullable()->after('ktp_kecamatan')->comment('Kota/Negara'); + $table->string('ktp_provinsi')->nullable()->after('town_country')->comment('Provinsi sesuai KTP'); + $table->string('post_code')->nullable()->after('ktp_provinsi')->comment('Kode pos alternatif'); + $table->string('l_dom_street')->nullable()->after('post_code')->comment('Alamat domisili - jalan'); + $table->string('l_dom_rt')->nullable()->after('l_dom_street')->comment('Alamat domisili - RT'); + $table->string('l_dom_kelurahan')->nullable()->after('l_dom_rt')->comment('Alamat domisili - kelurahan'); + $table->string('l_dom_rw')->nullable()->after('l_dom_kelurahan')->comment('Alamat domisili - RW'); + $table->string('l_dom_kecamatan')->nullable()->after('l_dom_rw')->comment('Alamat domisili - kecamatan'); + $table->string('l_dom_provinsi')->nullable()->after('l_dom_kecamatan')->comment('Alamat domisili - provinsi'); + $table->string('l_dom_t_country')->nullable()->after('l_dom_provinsi')->comment('Alamat domisili - kota/negara'); + $table->string('l_dom_post_code')->nullable()->after('l_dom_t_country')->comment('Alamat domisili - kode pos'); + }); + } + + /** + * Reverse the migrations. + * Menghapus field-field yang ditambahkan + */ + public function down(): void + { + Schema::table('customers', function (Blueprint $table) { + $table->dropColumn([ + 'ktp_kelurahan', + 'ktp_kecamatan', + 'town_country', + 'ktp_provinsi', + 'post_code', + 'l_dom_street', + 'l_dom_rt', + 'l_dom_kelurahan', + 'l_dom_rw', + 'l_dom_kecamatan', + 'l_dom_provinsi', + 'l_dom_t_country', + 'l_dom_post_code' + ]); + }); + } +};