arifal/sibedas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add column full pbg task payments

Browse Source
This commit is contained in:
arifal hidayat
2025-09-11 02:06:34 +07:00
parent 809eb85255
commit 53d12d6798
5 changed files with 561 additions and 180 deletions
Download Patch File Download Diff File Expand all files Collapse all files

483
app/Services/ServiceGoogleSheet.php
View File

@@ -13,6 +13,9 @@ use Exception;
use Google\Client as Google_Client;
use Google\Service\Sheets as Google_Service_Sheets;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Schema;
use App\Models\PbgTask;
class ServiceGoogleSheet
{
protected $client;
@@ -36,8 +39,8 @@ class ServiceGoogleSheet
public function run_service(){
try{
// $this->sync_big_data();
$this->sync_google_sheet_data();
$this->sync_pbg_task_payments();
}catch(Exception $e){
throw $e;
}
@@ -423,196 +426,384 @@ class ServiceGoogleSheet
}
}
public function get_realisasi_pad_data(){
/**
* Get sheet data where the first row is treated as headers, and subsequent rows
* are returned as associative arrays keyed by header names. Supports selecting
* a contiguous column range plus additional specific columns.
*
* Example: get_sheet_data_with_headers_range('Data', 'A', 'AX', ['BX'])
*
* @param string $sheet_name
* @param string $start_column_letter Inclusive start column letter (e.g., 'A')
* @param string $end_column_letter Inclusive end column letter (e.g., 'AX')
* @param array $extra_column_letters Additional discrete column letters (e.g., ['BX'])
* @return array{headers: array<int,string>, data: array<int,array<string,?string>>, selected_columns: array<int,int>}
*/
public function get_sheet_data_with_headers_range(string $sheet_name, string $start_column_letter, string $end_column_letter, array $extra_column_letters = [])
{
try {
// Get data from "REALISASI PAD" sheet
$sheet_data = $this->get_data_by_sheet_name("REALISASI PAD");
$sheet_data = $this->get_data_by_sheet_name($sheet_name);
if (empty($sheet_data)) {
Log::warning("No data found in REALISASI PAD sheet");
return [];
Log::warning("No data found in sheet", ['sheet_name' => $sheet_name]);
return [
'headers' => [],
'data' => [],
'selected_columns' => []
];
}
// Column indices: C=2, AK=36, AL=37, AW=48 (0-based)
$columns = [
'C' => 2,
'AK' => 36,
'AL' => 37,
'AW' => 48
];
// Build selected column indices: range A..AX and extras like BX
$selected_indices = $this->expandColumnRangeToIndices($start_column_letter, $end_column_letter);
foreach ($extra_column_letters as $letter) {
$selected_indices[] = $this->columnLetterToIndex($letter);
}
// Ensure unique and sorted
$selected_indices = array_values(array_unique($selected_indices));
sort($selected_indices);
$result = [
'headers' => [],
'data' => []
'data' => [],
'selected_columns' => $selected_indices
];
foreach ($sheet_data as $row_index => $row) {
if (!is_array($row)) continue;
if ($row_index === 0) {
// First row contains headers
foreach ($columns as $column_name => $column_index) {
$result['headers'][$column_name] = isset($row[$column_index]) ? trim($row[$column_index]) : '';
// First row contains headers (by selected columns)
foreach ($selected_indices as $col_index) {
$raw = isset($row[$col_index]) ? trim((string) $row[$col_index]) : '';
// Fallback to column letter if empty
$header = $raw !== '' ? $raw : $this->indexToColumnLetter($col_index);
$result['headers'][$col_index] = $this->normalizeHeader($header);
}
} else {
// Data rows
$row_data = [
'row' => $row_index + 1 // 1-based row number
];
$row_assoc = [];
$has_data = false;
foreach ($columns as $column_name => $column_index) {
$value = isset($row[$column_index]) ? trim($row[$column_index]) : '';
$row_data[$column_name] = $value;
foreach ($selected_indices as $col_index) {
$header = $result['headers'][$col_index] ?? $this->normalizeHeader($this->indexToColumnLetter($col_index));
$value = isset($row[$col_index]) ? trim((string) $row[$col_index]) : '';
$row_assoc[$header] = ($value === '') ? null : $value;
if ($value !== '') {
$has_data = true;
}
}
// Only add row if it has at least one non-empty value
if ($has_data) {
$result['data'][] = $row_data;
$result['data'][] = $row_assoc;
}
}
}
Log::info("REALISASI PAD Multiple Columns Data", [
'sheet_name' => 'REALISASI PAD',
'columns' => array_keys($columns),
'headers' => $result['headers'],
'total_data_rows' => count($result['data']),
'sample_data' => array_slice($result['data'], 0, 5), // Show first 5 rows as sample
'column_indices' => $columns
]);
return $result;
} catch (\Exception $e) {
Log::error("Error getting REALISASI PAD data", ['error' => $e->getMessage()]);
Log::error("Error getting sheet data with headers", [
'sheet_name' => $sheet_name,
'error' => $e->getMessage()
]);
throw $e;
}
}
/**
* Convert a column letter (e.g., 'A', 'Z', 'AA', 'AX', 'BX') to a zero-based index (A=0)
*/
private function columnLetterToIndex(string $letter): int
{
$letter = strtoupper(trim($letter));
$length = strlen($letter);
$index = 0;
for ($i = 0; $i < $length; $i++) {
$index = $index * 26 + (ord($letter[$i]) - ord('A') + 1);
}
return $index - 1; // zero-based
}
/**
* Convert zero-based column index to column letter (0='A')
*/
private function indexToColumnLetter(int $index): string
{
$index += 1; // make 1-based for calculation
$letters = '';
while ($index > 0) {
$mod = ($index - 1) % 26;
$letters = chr($mod + ord('A')) . $letters;
$index = intdiv($index - 1, 26);
}
return $letters;
}
/**
* Expand a column range like 'A'..'AX' to zero-based indices array
*/
private function expandColumnRangeToIndices(string $start_letter, string $end_letter): array
{
$start = $this->columnLetterToIndex($start_letter);
$end = $this->columnLetterToIndex($end_letter);
if ($start > $end) {
[$start, $end] = [$end, $start];
}
return range($start, $end);
}
/**
* Normalize header: trim, lowercase, replace spaces with underscore, remove non-alnum/underscore
*/
private function normalizeHeader(string $header): string
{
$header = trim($header);
$header = strtolower($header);
$header = preg_replace('/\s+/', '_', $header);
$header = preg_replace('/[^a-z0-9_]/', '', $header);
return $header;
}
public function sync_pbg_task_payments(){
try {
// Get payment data from REALISASI PAD sheet
$payment_data = $this->get_realisasi_pad_data();
if (empty($payment_data['data'])) {
Log::warning("No payment data found to sync");
return ['success' => false, 'message' => 'No payment data found'];
$sheetName = 'Data';
$startLetter = 'A';
$endLetter = 'AX';
$extraLetters = ['BF'];
// Fetch header row only (row 1) across A..BF and build header/selection
$headerRange = sprintf('%s!%s1:%s1', $sheetName, $startLetter, 'BF');
$headerResponse = $this->service->spreadsheets_values->get($this->spreadsheetID, $headerRange);
$headerRow = $headerResponse->getValues()[0] ?? [];
if (empty($headerRow)) {
Log::warning("No header row found in sheet", ['sheet' => $sheetName]);
return ['success' => false, 'message' => 'No header row found'];
}
$successful_syncs = 0;
$failed_syncs = 0;
$not_found_tasks = 0;
$not_found_tasks_registration_number = [];
$failed_sync_registration_numbers = [];
$errors = [];
// Selected indices: A..AX plus BF
$selected_indices = $this->expandColumnRangeToIndices($startLetter, $endLetter);
foreach ($extraLetters as $letter) {
$selected_indices[] = $this->columnLetterToIndex($letter);
}
$selected_indices = array_values(array_unique($selected_indices));
sort($selected_indices);
foreach ($payment_data['data'] as $row) {
try {
// Clean registration number from column C
$registration_number = \App\Models\PbgTaskPayment::cleanRegistrationNumber($row['C']);
if (empty($registration_number)) {
$failed_syncs++;
$failed_sync_registration_numbers[] = [
'row' => $row['row'],
'registration_number' => $row['C'] ?? 'EMPTY',
'reason' => 'Empty registration number'
];
continue;
}
// Find PBG task by registration number
$pbg_task = \App\Models\PbgTask::where('registration_number', $registration_number)->first();
if (!$pbg_task) {
$not_found_tasks_registration_number[] = [
'row' => $row['row'],
'registration_number' => $registration_number
];
$not_found_tasks++;
Log::warning("PBG Task not found for registration number", [
'registration_number' => $registration_number,
'row' => $row['row']
]);
continue;
}
// Convert data types
$payment_date = \App\Models\PbgTaskPayment::convertPaymentDate($row['AK']);
$pad_amount = \App\Models\PbgTaskPayment::convertPadAmount($row['AW']);
$sts_form_number = !empty($row['AL']) ? trim($row['AL']) : null;
// Create or update payment record
\App\Models\PbgTaskPayment::updateOrCreate(
[
'pbg_task_id' => $pbg_task->id,
'registration_number' => $registration_number
],
[
'pbg_task_uid' => $pbg_task->uuid,
'sts_form_number' => $sts_form_number,
'payment_date' => $payment_date,
'pad_amount' => $pad_amount
]
);
$successful_syncs++;
} catch (\Exception $e) {
$failed_syncs++;
$registration_number = $row['C'] ?? 'N/A';
$failed_sync_registration_numbers[] = [
'row' => $row['row'],
'registration_number' => $registration_number,
'reason' => $e->getMessage()
];
$errors[] = [
'row' => $row['row'],
'registration_number' => $registration_number,
'error' => $e->getMessage()
];
Log::error("Error syncing payment data for row", [
'row' => $row['row'],
'registration_number' => $registration_number,
'error' => $e->getMessage()
]);
}
// Build normalized headers map (index -> header)
$headers = [];
foreach ($selected_indices as $colIdx) {
$raw = isset($headerRow[$colIdx]) ? trim((string) $headerRow[$colIdx]) : '';
$header = $raw !== '' ? $raw : $this->indexToColumnLetter($colIdx);
$headers[$colIdx] = $this->normalizeHeader($header);
}
$result = [
'success' => true,
'total_rows' => count($payment_data['data']),
'successful_syncs' => $successful_syncs,
'failed_syncs' => $failed_syncs,
'not_found_tasks' => $not_found_tasks,
'not_found_tasks_registration_number' => $not_found_tasks_registration_number,
'failed_sync_registration_numbers' => $failed_sync_registration_numbers,
'errors' => $errors
// Truncate table and restart identity
Schema::disableForeignKeyConstraints();
DB::table('pbg_task_payments')->truncate();
Schema::enableForeignKeyConstraints();
// Map header -> db column
$map = [
'no' => 'row_no',
'jenis_konsultasi' => 'consultation_type',
'no_registrasi' => 'source_registration_number',
'nama_pemilik' => 'owner_name',
'lokasi_bg' => 'building_location',
'fungsi_bg' => 'building_function',
'nama_bangunan' => 'building_name',
'tgl_permohonan' => 'application_date_raw',
'status_verifikasi' => 'verification_status',
'status_permohonan' => 'application_status',
'alamat_pemilik' => 'owner_address',
'no_hp' => 'owner_phone',
'email' => 'owner_email',
'tanggal_catatan' => 'note_date_raw',
'catatan_kekurangan_dokumen' => 'document_shortage_note',
'gambar' => 'image_url',
'krkkkpr' => 'krk_kkpr',
'no_krk' => 'krk_number',
'lh' => 'lh',
'ska' => 'ska',
'keterangan' => 'remarks',
'helpdesk' => 'helpdesk',
'pj' => 'person_in_charge',
'operator_pbg' => 'pbg_operator',
'kepemilikan' => 'ownership',
'potensi_taru' => 'taru_potential',
'validasi_dinas' => 'agency_validation',
'kategori_retribusi' => 'retribution_category',
'no_urut_ba_tpt_20250001' => 'ba_tpt_number',
'tanggal_ba_tpt' => 'ba_tpt_date_raw',
'no_urut_ba_tpa' => 'ba_tpa_number',
'tanggal_ba_tpa' => 'ba_tpa_date_raw',
'no_urut_skrd_20250001' => 'skrd_number',
'tanggal_skrd' => 'skrd_date_raw',
'ptsp' => 'ptsp_status',
'selesai_terbit' => 'issued_status',
'tanggal_pembayaran_yyyymmdd' => 'payment_date_raw',
'format_sts' => 'sts_format',
'tahun_terbit' => 'issuance_year',
'tahun_berjalan' => 'current_year',
'kelurahan' => 'village',
'kecamatan' => 'district',
'lb' => 'building_area',
'tb' => 'building_height',
'jlb' => 'floor_count',
'unit' => 'unit_count',
'usulan_retribusi' => 'proposed_retribution',
'nilai_retribusi_keseluruhan_simbg' => 'retribution_total_simbg',
'nilai_retribusi_keseluruhan_pad' => 'retribution_total_pad',
'denda' => 'penalty_amount',
'usaha__non_usaha' => 'business_category',
];
Log::info("PBG Task Payments sync completed", $result);
// We'll build registration map lazily per chunk to limit memory
$regToTask = [];
// Log detailed arrays for failed syncs and not found registration numbers
if (!empty($failed_sync_registration_numbers)) {
Log::warning("Failed Sync Registration Numbers", [
'count' => count($failed_sync_registration_numbers),
'details' => $failed_sync_registration_numbers
]);
// Build and insert in small batches to avoid high memory usage
$batch = [];
$batchSize = 500;
$inserted = 0;
// Stream rows in chunks from API to avoid loading full sheet
$rowStart = 2; // data starts from row 2
$chunkRowSize = 800; // number of rows per chunk
$inserted = 0;
while (true) {
$rowEnd = $rowStart + $chunkRowSize - 1;
$range = sprintf('%s!%s%d:%s%d', $sheetName, $startLetter, $rowStart, 'BF', $rowEnd);
$resp = $this->service->spreadsheets_values->get($this->spreadsheetID, $range);
$values = $resp->getValues() ?? [];
if (empty($values)) {
break; // no more rows
}
// Preload registration map for this chunk
$chunkRegs = [];
foreach ($values as $row) {
foreach ($selected_indices as $colIdx) {
// find normalized header for this index
$h = $headers[$colIdx] ?? null;
if ($h === 'no_registrasi') {
$val = isset($row[$colIdx]) ? trim((string) $row[$colIdx]) : '';
if ($val !== '') { $chunkRegs[$val] = true; }
}
}
}
if (!empty($chunkRegs)) {
$keys = array_keys($chunkRegs);
$tasks = PbgTask::whereIn('registration_number', $keys)->get(['id','uuid','registration_number']);
foreach ($tasks as $task) {
$regToTask[trim($task->registration_number)] = ['id' => $task->id, 'uuid' => $task->uuid];
}
}
// Build and insert this chunk
$batch = [];
foreach ($values as $row) {
$record = [
'created_at' => now(),
'updated_at' => now(),
];
// Map row values by headers
$rowByHeader = [];
foreach ($selected_indices as $colIdx) {
$h = $headers[$colIdx] ?? null;
if ($h === null) continue;
$rowByHeader[$h] = isset($row[$colIdx]) ? trim((string) $row[$colIdx]) : null;
if ($rowByHeader[$h] === '') $rowByHeader[$h] = null;
}
// Skip if this row looks like a header row
$headerCheckKeys = ['no','jenis_konsultasi','no_registrasi'];
$headerMatches = 0;
foreach ($headerCheckKeys as $hk) {
if (!array_key_exists($hk, $rowByHeader)) { continue; }
$val = $rowByHeader[$hk];
if ($val === null) { continue; }
if ($this->normalizeHeader($val) === $hk) {
$headerMatches++;
}
}
if ($headerMatches >= 2) {
continue; // looks like a repeated header row, skip
}
// Skip if the entire row is empty (no values)
$hasAnyData = false;
foreach ($rowByHeader as $v) {
if ($v !== null && $v !== '') { $hasAnyData = true; break; }
}
if (!$hasAnyData) { continue; }
foreach ($map as $header => $column) {
$value = $rowByHeader[$header] ?? null;
switch ($column) {
case 'row_no':
case 'floor_count':
case 'unit_count':
case 'issuance_year':
case 'current_year':
$record[$column] = ($value === null || $value === '') ? null : (int) $value;
break;
case 'application_date_raw':
case 'note_date_raw':
case 'ba_tpt_date_raw':
case 'ba_tpa_date_raw':
case 'skrd_date_raw':
case 'payment_date_raw':
$record[$column] = $this->convertToDate($value);
break;
case 'building_area':
case 'building_height':
case 'proposed_retribution':
case 'retribution_total_simbg':
case 'retribution_total_pad':
case 'penalty_amount':
$record[$column] = $this->convertToDecimal($value);
break;
default:
if (is_string($value)) { $value = trim($value); }
$record[$column] = ($value === '' ? null : $value);
}
}
// Final trim pass
foreach ($record as $k => $v) {
if (is_string($v)) {
$t = trim($v);
$record[$k] = ($t === '') ? null : $t;
}
}
// Resolve relation
$sourceReg = $rowByHeader['no_registrasi'] ?? null;
if (is_string($sourceReg)) { $sourceReg = trim($sourceReg); }
if (!empty($sourceReg) && isset($regToTask[$sourceReg])) {
$record['pbg_task_id'] = $regToTask[$sourceReg]['id'];
$record['pbg_task_uid'] = $regToTask[$sourceReg]['uuid'];
} else {
$record['pbg_task_id'] = null;
$record['pbg_task_uid'] = null;
}
$batch[] = $record;
}
if (!empty($batch)) {
\App\Models\PbgTaskPayment::insert($batch);
$inserted += count($batch);
}
// next chunk
$rowStart = $rowEnd + 1;
if (function_exists('gc_collect_cycles')) { gc_collect_cycles(); }
}
if (!empty($not_found_tasks_registration_number)) {
Log::warning("Not Found Registration Numbers", [
'count' => count($not_found_tasks_registration_number),
'details' => $not_found_tasks_registration_number
]);
if (!empty($batch)) {
\App\Models\PbgTaskPayment::insert($batch);
$inserted += count($batch);
}
return $result;
Log::info('PBG Task Payments reloaded from sheet', ['inserted' => $inserted]);
return ['success' => true, 'inserted' => $inserted];
} catch (\Exception $e) {
Log::error("Error syncing PBG task payments", ['error' => $e->getMessage()]);