From 10fcdb5ea288998a9ae3a569eb14e4fd01f1a4d2 Mon Sep 17 00:00:00 2001 From: Daeng Deni Mardaeni Date: Wed, 4 Jun 2025 14:47:14 +0700 Subject: [PATCH] feat(webstatement): tambahkan job untuk ekspor data pernyataan periode ke CSV - Menambahkan **`ExportStatementPeriodJob`** untuk mendukung ekspor data pernyataan periode ke file CSV: - Inisialisasi job dengan parameter `account_number`, `period`, `saldo`, `client`, dan `disk`. - Menghitung tanggal mulai (`startDate`) dan tanggal akhir (`endDate`) berdasarkan periode yang diberikan. - Log informasi eksekusi termasuk waktu mulai, rentang tanggal, dan hasil akhir. - Menangani error saat proses berjalan dengan logging terperinci. - Memproses data pernyataan sebelum ekspor, termasuk: - Melakukan validasi jumlah data yang telah diproses sebelumnya. - Jika data belum sepenuhnya diproses, menghapus data lama pada tabel `processed_statements`. - Memproses data baru dengan mengolah entri dari tabel `StmtEntry`. - Membuat narrative dari data transaksi menggunakan model terkait `TempFundsTransfer`, `TempStmtNarrParam`, dan `TempStmtNarrFormat`. - Menambahkan logika untuk format tanggal transaksi dan narrasi: - Format tanggal transaksi melalui fungsi `formatTransactionDate` dan `formatActualDate`. - Narasi dihasilkan dari berbagai sumber, termasuk data transaksi dan parameter lainnya. - Ekspor data ke file CSV: - Mengatur struktur folder berdasarkan `client` dan `account_number`. - Menghapus file lama sebelum membuat file baru untuk menghindari duplikasi. - Membagi data menjadi chunks untuk mengurangi penggunaan memori. - Menambahkan header CSV dengan format kolom: `NO|TRANSACTION.DATE|REFERENCE.NUMBER|TRANSACTION.AMOUNT|TRANSACTION.TYPE|DESCRIPTION|END.BALANCE|ACTUAL.DATE`. - Optimasi proses: - Proses data dalam batch menggunakan pagination (`chunk`) untuk efisiensi memori. - Simpan hasil processed statement ke database sementara (`processed_statements`). - Tambahkan log setiap entri data untuk memonitor keberhasilan proses. - Tujuan penambahan ini: - Mendukung proses pengolahan dan pelaporan data rekening secara terstruktur. - Meningkatkan efisiensi penyimpanan dan akses data besar. - Menghasilkan file CSV yang dapat digunakan sebagai dokumen untuk laporan eksternal. Signed-off-by: Daeng Deni Mardaeni --- app/Jobs/ExportStatementPeriodJob.php | 443 ++++++++++++++++++++++++++ 1 file changed, 443 insertions(+) create mode 100644 app/Jobs/ExportStatementPeriodJob.php diff --git a/app/Jobs/ExportStatementPeriodJob.php b/app/Jobs/ExportStatementPeriodJob.php new file mode 100644 index 0000000..7d89bc9 --- /dev/null +++ b/app/Jobs/ExportStatementPeriodJob.php @@ -0,0 +1,443 @@ +account_number = $account_number; + $this->period = $period; + $this->saldo = $saldo; + $this->disk = $disk; + $this->client = $client; + $this->fileName = "{$account_number}_{$period}.csv"; + + // Calculate start and end dates based on period + $this->calculatePeriodDates(); + } + + /** + * Calculate start and end dates for the given period + */ + private function calculatePeriodDates(): void + { + $year = substr($this->period, 0, 4); + $month = substr($this->period, 4, 2); + + // Special case for May 2025 - start from 12th + if ($this->period === '202505') { + $this->startDate = Carbon::createFromDate($year, $month, 12)->startOfDay(); + } else { + // For all other periods, start from 1st of the month + $this->startDate = Carbon::createFromDate($year, $month, 1)->startOfDay(); + } + + // End date is always the last day of the month + $this->endDate = Carbon::createFromDate($year, $month, 1)->endOfMonth()->endOfDay(); + } + + /** + * Execute the job. + */ + public function handle(): void + { + try { + Log::info("Starting export statement period job for account: {$this->account_number}, period: {$this->period}"); + Log::info("Date range: {$this->startDate->format('Y-m-d')} to {$this->endDate->format('Y-m-d')}"); + + $this->processStatementData(); + $this->exportToCsv(); + + Log::info("Export statement period job completed successfully for account: {$this->account_number}, period: {$this->period}"); + } catch (Exception $e) { + Log::error("Error in ExportStatementPeriodJob: " . $e->getMessage()); + throw $e; + } + } + + private function processStatementData(): void + { + $accountQuery = [ + 'account_number' => $this->account_number, + 'period' => $this->period + ]; + + $totalCount = $this->getTotalEntryCount(); + $existingDataCount = $this->getExistingProcessedCount($accountQuery); + + // Only process if data is not fully processed + if ($existingDataCount !== $totalCount) { + $this->deleteExistingProcessedData($accountQuery); + $this->processAndSaveStatementEntries($totalCount); + } + } + + private function getTotalEntryCount(): int + { + return StmtEntry::where('account_number', $this->account_number) + ->whereBetween('date_time', [ + $this->startDate->format('ymdHi'), + $this->endDate->format('ymdHi') + ]) + ->count(); + } + + private function getExistingProcessedCount(array $criteria): int + { + return ProcessedStatement::where('account_number', $criteria['account_number']) + ->where('period', $criteria['period']) + ->count(); + } + + private function deleteExistingProcessedData(array $criteria): void + { + ProcessedStatement::where('account_number', $criteria['account_number']) + ->where('period', $criteria['period']) + ->delete(); + } + + private function processAndSaveStatementEntries(int $totalCount): void + { + $runningBalance = (float) $this->saldo; + $globalSequence = 0; + + Log::info("Processing {$totalCount} statement entries for account: {$this->account_number}"); + + StmtEntry::with(['ft', 'transaction']) + ->where('account_number', $this->account_number) + ->whereBetween('date_time', [ + $this->startDate->format('ymdHi'), + $this->endDate->format('ymdHi') + ]) + ->orderBy('date_time', 'ASC') + ->orderBy('trans_reference', 'ASC') + ->chunk($this->chunkSize, function ($entries) use (&$runningBalance, &$globalSequence) { + $processedData = $this->prepareProcessedData($entries, $runningBalance, $globalSequence); + + if (!empty($processedData)) { + DB::table('processed_statements')->insert($processedData); + } + }); + } + + private function prepareProcessedData($entries, &$runningBalance, &$globalSequence): array + { + $processedData = []; + + foreach ($entries as $item) { + $globalSequence++; + $runningBalance += (float) $item->amount_lcy; + + $transactionDate = $this->formatTransactionDate($item); + $actualDate = $this->formatActualDate($item); + + $processedData[] = [ + 'account_number' => $this->account_number, + 'period' => $this->period, + 'sequence_no' => $globalSequence, + 'transaction_date' => $transactionDate, + 'reference_number' => $item->trans_reference, + 'transaction_amount' => $item->amount_lcy, + 'transaction_type' => $item->amount_lcy < 0 ? 'D' : 'C', + 'description' => $this->generateNarrative($item), + 'end_balance' => $runningBalance, + 'actual_date' => $actualDate, + 'created_at' => now(), + 'updated_at' => now(), + ]; + } + + return $processedData; + } + + private function formatTransactionDate($item): string + { + try { + $prefix = substr($item->trans_reference ?? '', 0, 2); + $relationMap = [ + 'FT' => 'ft', + 'TT' => 'tt', + 'DC' => 'dc', + 'AA' => 'aa' + ]; + + $datetime = $item->date_time; + if (isset($relationMap[$prefix])) { + $relation = $relationMap[$prefix]; + $datetime = $item->$relation?->date_time ?? $datetime; + } + + // Extract date from datetime (first 6 characters) and time (last 4 characters) + $dateStr = substr($datetime, 0, 6); // YYMMDD + $timeStr = substr($datetime, 6, 4); // HHMM + + return Carbon::createFromFormat( + 'ymdHi', + $dateStr . $timeStr + )->format('d/m/Y H:i'); + } catch (Exception $e) { + Log::warning("Error formatting transaction date: " . $e->getMessage()); + return Carbon::now()->format('d/m/Y H:i'); + } + } + + private function formatActualDate($item): string + { + try { + $prefix = substr($item->trans_reference ?? '', 0, 2); + $relationMap = [ + 'FT' => 'ft', + 'TT' => 'tt', + 'DC' => 'dc', + 'AA' => 'aa' + ]; + + $datetime = $item->date_time; + if (isset($relationMap[$prefix])) { + $relation = $relationMap[$prefix]; + $datetime = $item->$relation?->date_time ?? $datetime; + } + + return Carbon::createFromFormat( + 'ymdHi', + $datetime + )->format('d/m/Y H:i'); + } catch (Exception $e) { + Log::warning("Error formatting actual date: " . $e->getMessage()); + return Carbon::now()->format('d/m/Y H:i'); + } + } + + /** + * Generate narrative for a statement entry + */ + private function generateNarrative($item) + { + $narr = []; + + if ($item->transaction) { + if ($item->transaction->stmt_narr) { + $narr[] = $item->transaction->stmt_narr; + } + if ($item->narrative) { + $narr[] = $item->narrative; + } + if ($item->transaction->narr_type) { + $narr[] = $this->getFormatNarrative($item->transaction->narr_type, $item); + } + } else if ($item->narrative) { + $narr[] = $item->narrative; + } + + if ($item->ft?->recipt_no) { + $narr[] = 'Receipt No: ' . $item->ft->recipt_no; + } + + return implode(' ', array_filter($narr)); + } + + /** + * Get formatted narrative based on narrative type + */ + private function getFormatNarrative($narr, $item) + { + $narrParam = TempStmtNarrParam::where('_id', $narr)->first(); + + if (!$narrParam) { + return ''; + } + + $fmt = ''; + if ($narrParam->_id == 'FTIN') { + $fmt = 'FT.IN'; + } else if ($narrParam->_id == 'FTOUT') { + $fmt = 'FT.OUT'; + } else if ($narrParam->_id == 'TTTRFOUT') { + $fmt = 'TT.O.TRF'; + } else if ($narrParam->_id == 'TTTRFIN') { + $fmt = 'TT.I.TRF'; + } else if ($narrParam->_id == 'APITRX'){ + $fmt = 'API.TSEL'; + } else if ($narrParam->_id == 'ONUSCR'){ + $fmt = 'ONUS.CR'; + } else if ($narrParam->_id == 'ONUSDR'){ + $fmt = 'ONUS.DR'; + }else { + $fmt = $narrParam->_id; + } + + $narrFormat = TempStmtNarrFormat::where('_id', $fmt)->first(); + + if (!$narrFormat) { + return ''; + } + + // Get the format string from the database + $formatString = $narrFormat->text_data ?? ''; + + // Parse the format string + // Split by the separator ']' + $parts = explode(']', $formatString); + + $result = ''; + + foreach ($parts as $index => $part) { + if (empty($part)) { + continue; + } + + if ($index === 0) { + // For the first part, take only what's before the '!' + $splitPart = explode('!', $part); + if (count($splitPart) > 0) { + // Remove quotes, backslashes, and other escape characters + $cleanPart = trim($splitPart[0]).' '; + // Remove quotes at the beginning and end + $cleanPart = preg_replace('/^["\'\\\\]+|["\'\\\\]+$/', '', $cleanPart); + // Remove any remaining backslashes + $cleanPart = str_replace('\\', '', $cleanPart); + // Remove any remaining quotes + $cleanPart = str_replace('"', '', $cleanPart); + $result .= $cleanPart; + } + } else { + // For other parts, these are field placeholders + $fieldName = strtolower(str_replace('.', '_', $part)); + + // Get the corresponding parameter value from narrParam + $paramValue = null; + + // Check if the field exists as a property in narrParam + if (property_exists($narrParam, $fieldName)) { + $paramValue = $narrParam->$fieldName; + } else if (isset($narrParam->$fieldName)) { + $paramValue = $narrParam->$fieldName; + } + + // If we found a value, add it to the result + if ($paramValue !== null) { + $result .= $paramValue; + } else { + // If no value found, try to use the original field name as a fallback + if ($fieldName !== 'recipt_no') { + $prefix = substr($item->trans_reference ?? '', 0, 2); + $relationMap = [ + 'FT' => 'ft', + 'TT' => 'tt', + 'DC' => 'dc', + 'AA' => 'aa' + ]; + + if (isset($relationMap[$prefix])) { + $relation = $relationMap[$prefix]; + $result .= ($item->$relation?->$fieldName ?? '') . ' '; + } + } + } + } + } + + return str_replace('', ' ', $result); + } + + /** + * Export processed data to CSV file + */ + private function exportToCsv(): void + { + // Determine the base path based on client + $basePath = !empty($this->client) + ? "statements/{$this->client}" + : "statements"; + + // Create client directory if it doesn't exist + if (!empty($this->client)) { + Storage::disk($this->disk)->makeDirectory($basePath); + } + + // Create account directory + $accountPath = "{$basePath}/{$this->account_number}"; + Storage::disk($this->disk)->makeDirectory($accountPath); + + $filePath = "{$accountPath}/{$this->fileName}"; + + // Delete existing file if it exists + if (Storage::disk($this->disk)->exists($filePath)) { + Storage::disk($this->disk)->delete($filePath); + } + + $csvContent = "NO|TRANSACTION.DATE|REFERENCE.NUMBER|TRANSACTION.AMOUNT|TRANSACTION.TYPE|DESCRIPTION|END.BALANCE|ACTUAL.DATE\n"; + + // Retrieve processed data in chunks to reduce memory usage + ProcessedStatement::where('account_number', $this->account_number) + ->where('period', $this->period) + ->orderBy('sequence_no') + ->chunk($this->chunkSize, function ($statements) use (&$csvContent, $filePath) { + foreach ($statements as $statement) { + $csvContent .= implode('|', [ + $statement->sequence_no, + $statement->transaction_date, + $statement->reference_number, + $statement->transaction_amount, + $statement->transaction_type, + $statement->description, + $statement->end_balance, + $statement->actual_date + ]) . "\n"; + } + + // Write to file incrementally to reduce memory usage + Storage::disk($this->disk)->append($filePath, $csvContent); + $csvContent = ''; // Reset content after writing + }); + + Log::info("Statement exported to {$this->disk} disk: {$filePath}"); + } + + /** + * Get transaction data by reference and field + */ + private function getTransaction($ref, $field) + { + $trans = TempFundsTransfer::where('ref_no', $ref)->first(); + return $trans->$field ?? ""; + } +}