add column full pbg task payments
This commit is contained in:
arifal hidayat
@@ -19,7 +19,7 @@ class SyncPbgTaskPayments extends Command
|
|||||||
*
|
*
|
||||||
* @var string
|
* @var string
|
||||||
*/
|
*/
|
||||||
protected $description = 'Sync PBG task payments from Google Sheets REALISASI PAD';
|
protected $description = 'Sync PBG task payments from Google Sheets Sheet Data';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Execute the console command.
|
* Execute the console command.
|
||||||
@@ -42,36 +42,13 @@ class SyncPbgTaskPayments extends Command
|
|||||||
$this->newLine();
|
$this->newLine();
|
||||||
|
|
||||||
$this->table(
|
$this->table(
|
||||||
['Metric', 'Count'],
|
['Metric', 'Value'],
|
||||||
[
|
[
|
||||||
['Total rows processed', $result['total_rows']],
|
['Inserted rows', $result['inserted'] ?? 0],
|
||||||
['Successful syncs', $result['successful_syncs']],
|
['Success', ($result['success'] ?? false) ? 'Yes' : 'No'],
|
||||||
['Failed syncs', $result['failed_syncs']],
|
|
||||||
['Tasks not found', $result['not_found_tasks']],
|
|
||||||
]
|
]
|
||||||
);
|
);
|
||||||
|
|
||||||
// Show success rate
|
|
||||||
$success_rate = $result['total_rows'] > 0
|
|
||||||
? round(($result['successful_syncs'] / $result['total_rows']) * 100, 2)
|
|
||||||
: 0;
|
|
||||||
|
|
||||||
$this->info("📈 Success rate: {$success_rate}%");
|
|
||||||
|
|
||||||
// Show errors if any
|
|
||||||
if (!empty($result['errors'])) {
|
|
||||||
$this->newLine();
|
|
||||||
$this->warn('⚠️ Errors encountered:');
|
|
||||||
foreach (array_slice($result['errors'], 0, 5) as $error) {
|
|
||||||
$this->line(" Row {$error['row']} ({$error['registration_number']}): {$error['error']}");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (count($result['errors']) > 5) {
|
|
||||||
$remaining = count($result['errors']) - 5;
|
|
||||||
$this->line(" ... and {$remaining} more errors (check logs for details)");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
$this->newLine();
|
$this->newLine();
|
||||||
$this->info('📝 Check Laravel logs for detailed information.');
|
$this->info('📝 Check Laravel logs for detailed information.');
|
||||||
|
|
||||||
|
|||||||
@@ -66,7 +66,7 @@ class ScrapingDataJob implements ShouldQueue
|
|||||||
Log::info("=== STEP 1: SCRAPING GOOGLE SHEET ===");
|
Log::info("=== STEP 1: SCRAPING GOOGLE SHEET ===");
|
||||||
$import_datasource->update(['message' => 'Scraping Google Sheet data...']);
|
$import_datasource->update(['message' => 'Scraping Google Sheet data...']);
|
||||||
|
|
||||||
// $service_google_sheet->run_service();
|
$service_google_sheet->run_service();
|
||||||
Log::info("Google Sheet scraping completed successfully");
|
Log::info("Google Sheet scraping completed successfully");
|
||||||
|
|
||||||
// STEP 2: Scrape PBG Task to get parent data
|
// STEP 2: Scrape PBG Task to get parent data
|
||||||
|
|||||||
@@ -11,15 +11,79 @@ class PbgTaskPayment extends Model
|
|||||||
protected $fillable = [
|
protected $fillable = [
|
||||||
'pbg_task_id',
|
'pbg_task_id',
|
||||||
'pbg_task_uid',
|
'pbg_task_uid',
|
||||||
'registration_number',
|
// mapped fields
|
||||||
'sts_form_number',
|
'row_no',
|
||||||
'payment_date',
|
'consultation_type',
|
||||||
'pad_amount'
|
'source_registration_number',
|
||||||
|
'owner_name',
|
||||||
|
'building_location',
|
||||||
|
'building_function',
|
||||||
|
'building_name',
|
||||||
|
'application_date_raw',
|
||||||
|
'verification_status',
|
||||||
|
'application_status',
|
||||||
|
'owner_address',
|
||||||
|
'owner_phone',
|
||||||
|
'owner_email',
|
||||||
|
'note_date_raw',
|
||||||
|
'document_shortage_note',
|
||||||
|
'image_url',
|
||||||
|
'krk_kkpr',
|
||||||
|
'krk_number',
|
||||||
|
'lh',
|
||||||
|
'ska',
|
||||||
|
'remarks',
|
||||||
|
'helpdesk',
|
||||||
|
'person_in_charge',
|
||||||
|
'pbg_operator',
|
||||||
|
'ownership',
|
||||||
|
'taru_potential',
|
||||||
|
'agency_validation',
|
||||||
|
'retribution_category',
|
||||||
|
'ba_tpt_number',
|
||||||
|
'ba_tpt_date_raw',
|
||||||
|
'ba_tpa_number',
|
||||||
|
'ba_tpa_date_raw',
|
||||||
|
'skrd_number',
|
||||||
|
'skrd_date_raw',
|
||||||
|
'ptsp_status',
|
||||||
|
'issued_status',
|
||||||
|
'payment_date_raw',
|
||||||
|
'sts_format',
|
||||||
|
'issuance_year',
|
||||||
|
'current_year',
|
||||||
|
'village',
|
||||||
|
'district',
|
||||||
|
'building_area',
|
||||||
|
'building_height',
|
||||||
|
'floor_count',
|
||||||
|
'unit_count',
|
||||||
|
'proposed_retribution',
|
||||||
|
'retribution_total_simbg',
|
||||||
|
'retribution_total_pad',
|
||||||
|
'penalty_amount',
|
||||||
|
'business_category',
|
||||||
|
'created_at',
|
||||||
|
'updated_at'
|
||||||
];
|
];
|
||||||
|
|
||||||
protected $casts = [
|
protected $casts = [
|
||||||
'payment_date' => 'date',
|
'application_date_raw' => 'date',
|
||||||
'pad_amount' => 'decimal:2'
|
'note_date_raw' => 'date',
|
||||||
|
'ba_tpt_date_raw' => 'date',
|
||||||
|
'ba_tpa_date_raw' => 'date',
|
||||||
|
'skrd_date_raw' => 'date',
|
||||||
|
'payment_date_raw' => 'date',
|
||||||
|
'issuance_year' => 'integer',
|
||||||
|
'current_year' => 'integer',
|
||||||
|
'floor_count' => 'integer',
|
||||||
|
'unit_count' => 'integer',
|
||||||
|
'building_area' => 'decimal:2',
|
||||||
|
'building_height' => 'decimal:2',
|
||||||
|
'proposed_retribution' => 'decimal:2',
|
||||||
|
'retribution_total_simbg' => 'decimal:2',
|
||||||
|
'retribution_total_pad' => 'decimal:2',
|
||||||
|
'penalty_amount' => 'decimal:2'
|
||||||
];
|
];
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -13,6 +13,9 @@ use Exception;
|
|||||||
use Google\Client as Google_Client;
|
use Google\Client as Google_Client;
|
||||||
use Google\Service\Sheets as Google_Service_Sheets;
|
use Google\Service\Sheets as Google_Service_Sheets;
|
||||||
use Illuminate\Support\Facades\Log;
|
use Illuminate\Support\Facades\Log;
|
||||||
|
use Illuminate\Support\Facades\DB;
|
||||||
|
use Illuminate\Support\Facades\Schema;
|
||||||
|
use App\Models\PbgTask;
|
||||||
class ServiceGoogleSheet
|
class ServiceGoogleSheet
|
||||||
{
|
{
|
||||||
protected $client;
|
protected $client;
|
||||||
@@ -36,8 +39,8 @@ class ServiceGoogleSheet
|
|||||||
|
|
||||||
public function run_service(){
|
public function run_service(){
|
||||||
try{
|
try{
|
||||||
// $this->sync_big_data();
|
|
||||||
$this->sync_google_sheet_data();
|
$this->sync_google_sheet_data();
|
||||||
|
$this->sync_pbg_task_payments();
|
||||||
}catch(Exception $e){
|
}catch(Exception $e){
|
||||||
throw $e;
|
throw $e;
|
||||||
}
|
}
|
||||||
@@ -423,196 +426,384 @@ class ServiceGoogleSheet
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public function get_realisasi_pad_data(){
|
/**
|
||||||
|
* Get sheet data where the first row is treated as headers, and subsequent rows
|
||||||
|
* are returned as associative arrays keyed by header names. Supports selecting
|
||||||
|
* a contiguous column range plus additional specific columns.
|
||||||
|
*
|
||||||
|
* Example: get_sheet_data_with_headers_range('Data', 'A', 'AX', ['BX'])
|
||||||
|
*
|
||||||
|
* @param string $sheet_name
|
||||||
|
* @param string $start_column_letter Inclusive start column letter (e.g., 'A')
|
||||||
|
* @param string $end_column_letter Inclusive end column letter (e.g., 'AX')
|
||||||
|
* @param array $extra_column_letters Additional discrete column letters (e.g., ['BX'])
|
||||||
|
* @return array{headers: array<int,string>, data: array<int,array<string,?string>>, selected_columns: array<int,int>}
|
||||||
|
*/
|
||||||
|
public function get_sheet_data_with_headers_range(string $sheet_name, string $start_column_letter, string $end_column_letter, array $extra_column_letters = [])
|
||||||
|
{
|
||||||
try {
|
try {
|
||||||
// Get data from "REALISASI PAD" sheet
|
$sheet_data = $this->get_data_by_sheet_name($sheet_name);
|
||||||
$sheet_data = $this->get_data_by_sheet_name("REALISASI PAD");
|
|
||||||
|
|
||||||
if (empty($sheet_data)) {
|
if (empty($sheet_data)) {
|
||||||
Log::warning("No data found in REALISASI PAD sheet");
|
Log::warning("No data found in sheet", ['sheet_name' => $sheet_name]);
|
||||||
return [];
|
return [
|
||||||
|
'headers' => [],
|
||||||
|
'data' => [],
|
||||||
|
'selected_columns' => []
|
||||||
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
// Column indices: C=2, AK=36, AL=37, AW=48 (0-based)
|
// Build selected column indices: range A..AX and extras like BX
|
||||||
$columns = [
|
$selected_indices = $this->expandColumnRangeToIndices($start_column_letter, $end_column_letter);
|
||||||
'C' => 2,
|
foreach ($extra_column_letters as $letter) {
|
||||||
'AK' => 36,
|
$selected_indices[] = $this->columnLetterToIndex($letter);
|
||||||
'AL' => 37,
|
}
|
||||||
'AW' => 48
|
// Ensure unique and sorted
|
||||||
];
|
$selected_indices = array_values(array_unique($selected_indices));
|
||||||
|
sort($selected_indices);
|
||||||
|
|
||||||
$result = [
|
$result = [
|
||||||
'headers' => [],
|
'headers' => [],
|
||||||
'data' => []
|
'data' => [],
|
||||||
|
'selected_columns' => $selected_indices
|
||||||
];
|
];
|
||||||
|
|
||||||
foreach ($sheet_data as $row_index => $row) {
|
foreach ($sheet_data as $row_index => $row) {
|
||||||
if (!is_array($row)) continue;
|
if (!is_array($row)) continue;
|
||||||
|
|
||||||
if ($row_index === 0) {
|
if ($row_index === 0) {
|
||||||
// First row contains headers
|
// First row contains headers (by selected columns)
|
||||||
foreach ($columns as $column_name => $column_index) {
|
foreach ($selected_indices as $col_index) {
|
||||||
$result['headers'][$column_name] = isset($row[$column_index]) ? trim($row[$column_index]) : '';
|
$raw = isset($row[$col_index]) ? trim((string) $row[$col_index]) : '';
|
||||||
|
// Fallback to column letter if empty
|
||||||
|
$header = $raw !== '' ? $raw : $this->indexToColumnLetter($col_index);
|
||||||
|
$result['headers'][$col_index] = $this->normalizeHeader($header);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Data rows
|
$row_assoc = [];
|
||||||
$row_data = [
|
|
||||||
'row' => $row_index + 1 // 1-based row number
|
|
||||||
];
|
|
||||||
|
|
||||||
$has_data = false;
|
$has_data = false;
|
||||||
foreach ($columns as $column_name => $column_index) {
|
foreach ($selected_indices as $col_index) {
|
||||||
$value = isset($row[$column_index]) ? trim($row[$column_index]) : '';
|
$header = $result['headers'][$col_index] ?? $this->normalizeHeader($this->indexToColumnLetter($col_index));
|
||||||
$row_data[$column_name] = $value;
|
$value = isset($row[$col_index]) ? trim((string) $row[$col_index]) : '';
|
||||||
|
$row_assoc[$header] = ($value === '') ? null : $value;
|
||||||
if ($value !== '') {
|
if ($value !== '') {
|
||||||
$has_data = true;
|
$has_data = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Only add row if it has at least one non-empty value
|
|
||||||
if ($has_data) {
|
if ($has_data) {
|
||||||
$result['data'][] = $row_data;
|
$result['data'][] = $row_assoc;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Log::info("REALISASI PAD Multiple Columns Data", [
|
|
||||||
'sheet_name' => 'REALISASI PAD',
|
|
||||||
'columns' => array_keys($columns),
|
|
||||||
'headers' => $result['headers'],
|
|
||||||
'total_data_rows' => count($result['data']),
|
|
||||||
'sample_data' => array_slice($result['data'], 0, 5), // Show first 5 rows as sample
|
|
||||||
'column_indices' => $columns
|
|
||||||
]);
|
|
||||||
|
|
||||||
return $result;
|
return $result;
|
||||||
|
|
||||||
} catch (\Exception $e) {
|
} catch (\Exception $e) {
|
||||||
Log::error("Error getting REALISASI PAD data", ['error' => $e->getMessage()]);
|
Log::error("Error getting sheet data with headers", [
|
||||||
|
'sheet_name' => $sheet_name,
|
||||||
|
'error' => $e->getMessage()
|
||||||
|
]);
|
||||||
throw $e;
|
throw $e;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Convert a column letter (e.g., 'A', 'Z', 'AA', 'AX', 'BX') to a zero-based index (A=0)
|
||||||
|
*/
|
||||||
|
private function columnLetterToIndex(string $letter): int
|
||||||
|
{
|
||||||
|
$letter = strtoupper(trim($letter));
|
||||||
|
$length = strlen($letter);
|
||||||
|
$index = 0;
|
||||||
|
for ($i = 0; $i < $length; $i++) {
|
||||||
|
$index = $index * 26 + (ord($letter[$i]) - ord('A') + 1);
|
||||||
|
}
|
||||||
|
return $index - 1; // zero-based
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Convert zero-based column index to column letter (0='A')
|
||||||
|
*/
|
||||||
|
private function indexToColumnLetter(int $index): string
|
||||||
|
{
|
||||||
|
$index += 1; // make 1-based for calculation
|
||||||
|
$letters = '';
|
||||||
|
while ($index > 0) {
|
||||||
|
$mod = ($index - 1) % 26;
|
||||||
|
$letters = chr($mod + ord('A')) . $letters;
|
||||||
|
$index = intdiv($index - 1, 26);
|
||||||
|
}
|
||||||
|
return $letters;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Expand a column range like 'A'..'AX' to zero-based indices array
|
||||||
|
*/
|
||||||
|
private function expandColumnRangeToIndices(string $start_letter, string $end_letter): array
|
||||||
|
{
|
||||||
|
$start = $this->columnLetterToIndex($start_letter);
|
||||||
|
$end = $this->columnLetterToIndex($end_letter);
|
||||||
|
if ($start > $end) {
|
||||||
|
[$start, $end] = [$end, $start];
|
||||||
|
}
|
||||||
|
return range($start, $end);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Normalize header: trim, lowercase, replace spaces with underscore, remove non-alnum/underscore
|
||||||
|
*/
|
||||||
|
private function normalizeHeader(string $header): string
|
||||||
|
{
|
||||||
|
$header = trim($header);
|
||||||
|
$header = strtolower($header);
|
||||||
|
$header = preg_replace('/\s+/', '_', $header);
|
||||||
|
$header = preg_replace('/[^a-z0-9_]/', '', $header);
|
||||||
|
return $header;
|
||||||
|
}
|
||||||
|
|
||||||
public function sync_pbg_task_payments(){
|
public function sync_pbg_task_payments(){
|
||||||
try {
|
try {
|
||||||
// Get payment data from REALISASI PAD sheet
|
$sheetName = 'Data';
|
||||||
$payment_data = $this->get_realisasi_pad_data();
|
$startLetter = 'A';
|
||||||
|
$endLetter = 'AX';
|
||||||
|
$extraLetters = ['BF'];
|
||||||
|
|
||||||
if (empty($payment_data['data'])) {
|
// Fetch header row only (row 1) across A..BF and build header/selection
|
||||||
Log::warning("No payment data found to sync");
|
$headerRange = sprintf('%s!%s1:%s1', $sheetName, $startLetter, 'BF');
|
||||||
return ['success' => false, 'message' => 'No payment data found'];
|
$headerResponse = $this->service->spreadsheets_values->get($this->spreadsheetID, $headerRange);
|
||||||
|
$headerRow = $headerResponse->getValues()[0] ?? [];
|
||||||
|
if (empty($headerRow)) {
|
||||||
|
Log::warning("No header row found in sheet", ['sheet' => $sheetName]);
|
||||||
|
return ['success' => false, 'message' => 'No header row found'];
|
||||||
}
|
}
|
||||||
|
|
||||||
$successful_syncs = 0;
|
// Selected indices: A..AX plus BF
|
||||||
$failed_syncs = 0;
|
$selected_indices = $this->expandColumnRangeToIndices($startLetter, $endLetter);
|
||||||
$not_found_tasks = 0;
|
foreach ($extraLetters as $letter) {
|
||||||
$not_found_tasks_registration_number = [];
|
$selected_indices[] = $this->columnLetterToIndex($letter);
|
||||||
$failed_sync_registration_numbers = [];
|
}
|
||||||
$errors = [];
|
$selected_indices = array_values(array_unique($selected_indices));
|
||||||
|
sort($selected_indices);
|
||||||
|
|
||||||
foreach ($payment_data['data'] as $row) {
|
// Build normalized headers map (index -> header)
|
||||||
try {
|
$headers = [];
|
||||||
// Clean registration number from column C
|
foreach ($selected_indices as $colIdx) {
|
||||||
$registration_number = \App\Models\PbgTaskPayment::cleanRegistrationNumber($row['C']);
|
$raw = isset($headerRow[$colIdx]) ? trim((string) $headerRow[$colIdx]) : '';
|
||||||
|
$header = $raw !== '' ? $raw : $this->indexToColumnLetter($colIdx);
|
||||||
if (empty($registration_number)) {
|
$headers[$colIdx] = $this->normalizeHeader($header);
|
||||||
$failed_syncs++;
|
|
||||||
$failed_sync_registration_numbers[] = [
|
|
||||||
'row' => $row['row'],
|
|
||||||
'registration_number' => $row['C'] ?? 'EMPTY',
|
|
||||||
'reason' => 'Empty registration number'
|
|
||||||
];
|
|
||||||
continue;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Find PBG task by registration number
|
// Truncate table and restart identity
|
||||||
$pbg_task = \App\Models\PbgTask::where('registration_number', $registration_number)->first();
|
Schema::disableForeignKeyConstraints();
|
||||||
|
DB::table('pbg_task_payments')->truncate();
|
||||||
|
Schema::enableForeignKeyConstraints();
|
||||||
|
|
||||||
if (!$pbg_task) {
|
// Map header -> db column
|
||||||
$not_found_tasks_registration_number[] = [
|
$map = [
|
||||||
'row' => $row['row'],
|
'no' => 'row_no',
|
||||||
'registration_number' => $registration_number
|
'jenis_konsultasi' => 'consultation_type',
|
||||||
];
|
'no_registrasi' => 'source_registration_number',
|
||||||
$not_found_tasks++;
|
'nama_pemilik' => 'owner_name',
|
||||||
Log::warning("PBG Task not found for registration number", [
|
'lokasi_bg' => 'building_location',
|
||||||
'registration_number' => $registration_number,
|
'fungsi_bg' => 'building_function',
|
||||||
'row' => $row['row']
|
'nama_bangunan' => 'building_name',
|
||||||
]);
|
'tgl_permohonan' => 'application_date_raw',
|
||||||
continue;
|
'status_verifikasi' => 'verification_status',
|
||||||
}
|
'status_permohonan' => 'application_status',
|
||||||
|
'alamat_pemilik' => 'owner_address',
|
||||||
// Convert data types
|
'no_hp' => 'owner_phone',
|
||||||
$payment_date = \App\Models\PbgTaskPayment::convertPaymentDate($row['AK']);
|
'email' => 'owner_email',
|
||||||
$pad_amount = \App\Models\PbgTaskPayment::convertPadAmount($row['AW']);
|
'tanggal_catatan' => 'note_date_raw',
|
||||||
$sts_form_number = !empty($row['AL']) ? trim($row['AL']) : null;
|
'catatan_kekurangan_dokumen' => 'document_shortage_note',
|
||||||
|
'gambar' => 'image_url',
|
||||||
// Create or update payment record
|
'krkkkpr' => 'krk_kkpr',
|
||||||
\App\Models\PbgTaskPayment::updateOrCreate(
|
'no_krk' => 'krk_number',
|
||||||
[
|
'lh' => 'lh',
|
||||||
'pbg_task_id' => $pbg_task->id,
|
'ska' => 'ska',
|
||||||
'registration_number' => $registration_number
|
'keterangan' => 'remarks',
|
||||||
],
|
'helpdesk' => 'helpdesk',
|
||||||
[
|
'pj' => 'person_in_charge',
|
||||||
'pbg_task_uid' => $pbg_task->uuid,
|
'operator_pbg' => 'pbg_operator',
|
||||||
'sts_form_number' => $sts_form_number,
|
'kepemilikan' => 'ownership',
|
||||||
'payment_date' => $payment_date,
|
'potensi_taru' => 'taru_potential',
|
||||||
'pad_amount' => $pad_amount
|
'validasi_dinas' => 'agency_validation',
|
||||||
]
|
'kategori_retribusi' => 'retribution_category',
|
||||||
);
|
'no_urut_ba_tpt_20250001' => 'ba_tpt_number',
|
||||||
|
'tanggal_ba_tpt' => 'ba_tpt_date_raw',
|
||||||
$successful_syncs++;
|
'no_urut_ba_tpa' => 'ba_tpa_number',
|
||||||
|
'tanggal_ba_tpa' => 'ba_tpa_date_raw',
|
||||||
} catch (\Exception $e) {
|
'no_urut_skrd_20250001' => 'skrd_number',
|
||||||
$failed_syncs++;
|
'tanggal_skrd' => 'skrd_date_raw',
|
||||||
$registration_number = $row['C'] ?? 'N/A';
|
'ptsp' => 'ptsp_status',
|
||||||
$failed_sync_registration_numbers[] = [
|
'selesai_terbit' => 'issued_status',
|
||||||
'row' => $row['row'],
|
'tanggal_pembayaran_yyyymmdd' => 'payment_date_raw',
|
||||||
'registration_number' => $registration_number,
|
'format_sts' => 'sts_format',
|
||||||
'reason' => $e->getMessage()
|
'tahun_terbit' => 'issuance_year',
|
||||||
];
|
'tahun_berjalan' => 'current_year',
|
||||||
$errors[] = [
|
'kelurahan' => 'village',
|
||||||
'row' => $row['row'],
|
'kecamatan' => 'district',
|
||||||
'registration_number' => $registration_number,
|
'lb' => 'building_area',
|
||||||
'error' => $e->getMessage()
|
'tb' => 'building_height',
|
||||||
|
'jlb' => 'floor_count',
|
||||||
|
'unit' => 'unit_count',
|
||||||
|
'usulan_retribusi' => 'proposed_retribution',
|
||||||
|
'nilai_retribusi_keseluruhan_simbg' => 'retribution_total_simbg',
|
||||||
|
'nilai_retribusi_keseluruhan_pad' => 'retribution_total_pad',
|
||||||
|
'denda' => 'penalty_amount',
|
||||||
|
'usaha__non_usaha' => 'business_category',
|
||||||
];
|
];
|
||||||
|
|
||||||
Log::error("Error syncing payment data for row", [
|
// We'll build registration map lazily per chunk to limit memory
|
||||||
'row' => $row['row'],
|
$regToTask = [];
|
||||||
'registration_number' => $registration_number,
|
|
||||||
'error' => $e->getMessage()
|
// Build and insert in small batches to avoid high memory usage
|
||||||
]);
|
$batch = [];
|
||||||
|
$batchSize = 500;
|
||||||
|
$inserted = 0;
|
||||||
|
// Stream rows in chunks from API to avoid loading full sheet
|
||||||
|
$rowStart = 2; // data starts from row 2
|
||||||
|
$chunkRowSize = 800; // number of rows per chunk
|
||||||
|
$inserted = 0;
|
||||||
|
while (true) {
|
||||||
|
$rowEnd = $rowStart + $chunkRowSize - 1;
|
||||||
|
$range = sprintf('%s!%s%d:%s%d', $sheetName, $startLetter, $rowStart, 'BF', $rowEnd);
|
||||||
|
$resp = $this->service->spreadsheets_values->get($this->spreadsheetID, $range);
|
||||||
|
$values = $resp->getValues() ?? [];
|
||||||
|
if (empty($values)) {
|
||||||
|
break; // no more rows
|
||||||
|
}
|
||||||
|
|
||||||
|
// Preload registration map for this chunk
|
||||||
|
$chunkRegs = [];
|
||||||
|
foreach ($values as $row) {
|
||||||
|
foreach ($selected_indices as $colIdx) {
|
||||||
|
// find normalized header for this index
|
||||||
|
$h = $headers[$colIdx] ?? null;
|
||||||
|
if ($h === 'no_registrasi') {
|
||||||
|
$val = isset($row[$colIdx]) ? trim((string) $row[$colIdx]) : '';
|
||||||
|
if ($val !== '') { $chunkRegs[$val] = true; }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!empty($chunkRegs)) {
|
||||||
|
$keys = array_keys($chunkRegs);
|
||||||
|
$tasks = PbgTask::whereIn('registration_number', $keys)->get(['id','uuid','registration_number']);
|
||||||
|
foreach ($tasks as $task) {
|
||||||
|
$regToTask[trim($task->registration_number)] = ['id' => $task->id, 'uuid' => $task->uuid];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
$result = [
|
// Build and insert this chunk
|
||||||
'success' => true,
|
$batch = [];
|
||||||
'total_rows' => count($payment_data['data']),
|
foreach ($values as $row) {
|
||||||
'successful_syncs' => $successful_syncs,
|
$record = [
|
||||||
'failed_syncs' => $failed_syncs,
|
'created_at' => now(),
|
||||||
'not_found_tasks' => $not_found_tasks,
|
'updated_at' => now(),
|
||||||
'not_found_tasks_registration_number' => $not_found_tasks_registration_number,
|
|
||||||
'failed_sync_registration_numbers' => $failed_sync_registration_numbers,
|
|
||||||
'errors' => $errors
|
|
||||||
];
|
];
|
||||||
|
|
||||||
Log::info("PBG Task Payments sync completed", $result);
|
// Map row values by headers
|
||||||
|
$rowByHeader = [];
|
||||||
// Log detailed arrays for failed syncs and not found registration numbers
|
foreach ($selected_indices as $colIdx) {
|
||||||
if (!empty($failed_sync_registration_numbers)) {
|
$h = $headers[$colIdx] ?? null;
|
||||||
Log::warning("Failed Sync Registration Numbers", [
|
if ($h === null) continue;
|
||||||
'count' => count($failed_sync_registration_numbers),
|
$rowByHeader[$h] = isset($row[$colIdx]) ? trim((string) $row[$colIdx]) : null;
|
||||||
'details' => $failed_sync_registration_numbers
|
if ($rowByHeader[$h] === '') $rowByHeader[$h] = null;
|
||||||
]);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!empty($not_found_tasks_registration_number)) {
|
// Skip if this row looks like a header row
|
||||||
Log::warning("Not Found Registration Numbers", [
|
$headerCheckKeys = ['no','jenis_konsultasi','no_registrasi'];
|
||||||
'count' => count($not_found_tasks_registration_number),
|
$headerMatches = 0;
|
||||||
'details' => $not_found_tasks_registration_number
|
foreach ($headerCheckKeys as $hk) {
|
||||||
]);
|
if (!array_key_exists($hk, $rowByHeader)) { continue; }
|
||||||
|
$val = $rowByHeader[$hk];
|
||||||
|
if ($val === null) { continue; }
|
||||||
|
if ($this->normalizeHeader($val) === $hk) {
|
||||||
|
$headerMatches++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ($headerMatches >= 2) {
|
||||||
|
continue; // looks like a repeated header row, skip
|
||||||
}
|
}
|
||||||
|
|
||||||
return $result;
|
// Skip if the entire row is empty (no values)
|
||||||
|
$hasAnyData = false;
|
||||||
|
foreach ($rowByHeader as $v) {
|
||||||
|
if ($v !== null && $v !== '') { $hasAnyData = true; break; }
|
||||||
|
}
|
||||||
|
if (!$hasAnyData) { continue; }
|
||||||
|
|
||||||
|
foreach ($map as $header => $column) {
|
||||||
|
$value = $rowByHeader[$header] ?? null;
|
||||||
|
|
||||||
|
switch ($column) {
|
||||||
|
case 'row_no':
|
||||||
|
case 'floor_count':
|
||||||
|
case 'unit_count':
|
||||||
|
case 'issuance_year':
|
||||||
|
case 'current_year':
|
||||||
|
$record[$column] = ($value === null || $value === '') ? null : (int) $value;
|
||||||
|
break;
|
||||||
|
case 'application_date_raw':
|
||||||
|
case 'note_date_raw':
|
||||||
|
case 'ba_tpt_date_raw':
|
||||||
|
case 'ba_tpa_date_raw':
|
||||||
|
case 'skrd_date_raw':
|
||||||
|
case 'payment_date_raw':
|
||||||
|
$record[$column] = $this->convertToDate($value);
|
||||||
|
break;
|
||||||
|
case 'building_area':
|
||||||
|
case 'building_height':
|
||||||
|
case 'proposed_retribution':
|
||||||
|
case 'retribution_total_simbg':
|
||||||
|
case 'retribution_total_pad':
|
||||||
|
case 'penalty_amount':
|
||||||
|
$record[$column] = $this->convertToDecimal($value);
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
if (is_string($value)) { $value = trim($value); }
|
||||||
|
$record[$column] = ($value === '' ? null : $value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Final trim pass
|
||||||
|
foreach ($record as $k => $v) {
|
||||||
|
if (is_string($v)) {
|
||||||
|
$t = trim($v);
|
||||||
|
$record[$k] = ($t === '') ? null : $t;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Resolve relation
|
||||||
|
$sourceReg = $rowByHeader['no_registrasi'] ?? null;
|
||||||
|
if (is_string($sourceReg)) { $sourceReg = trim($sourceReg); }
|
||||||
|
if (!empty($sourceReg) && isset($regToTask[$sourceReg])) {
|
||||||
|
$record['pbg_task_id'] = $regToTask[$sourceReg]['id'];
|
||||||
|
$record['pbg_task_uid'] = $regToTask[$sourceReg]['uuid'];
|
||||||
|
} else {
|
||||||
|
$record['pbg_task_id'] = null;
|
||||||
|
$record['pbg_task_uid'] = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
$batch[] = $record;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!empty($batch)) {
|
||||||
|
\App\Models\PbgTaskPayment::insert($batch);
|
||||||
|
$inserted += count($batch);
|
||||||
|
}
|
||||||
|
|
||||||
|
// next chunk
|
||||||
|
$rowStart = $rowEnd + 1;
|
||||||
|
if (function_exists('gc_collect_cycles')) { gc_collect_cycles(); }
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!empty($batch)) {
|
||||||
|
\App\Models\PbgTaskPayment::insert($batch);
|
||||||
|
$inserted += count($batch);
|
||||||
|
}
|
||||||
|
|
||||||
|
Log::info('PBG Task Payments reloaded from sheet', ['inserted' => $inserted]);
|
||||||
|
|
||||||
|
return ['success' => true, 'inserted' => $inserted];
|
||||||
|
|
||||||
} catch (\Exception $e) {
|
} catch (\Exception $e) {
|
||||||
Log::error("Error syncing PBG task payments", ['error' => $e->getMessage()]);
|
Log::error("Error syncing PBG task payments", ['error' => $e->getMessage()]);
|
||||||
|
|||||||
@@ -0,0 +1,149 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
use Illuminate\Database\Migrations\Migration;
|
||||||
|
use Illuminate\Database\Schema\Blueprint;
|
||||||
|
use Illuminate\Support\Facades\Schema;
|
||||||
|
|
||||||
|
return new class extends Migration
|
||||||
|
{
|
||||||
|
/**
|
||||||
|
* Run the migrations.
|
||||||
|
*/
|
||||||
|
public function up(): void
|
||||||
|
{
|
||||||
|
Schema::table('pbg_task_payments', function (Blueprint $table) {
|
||||||
|
// Drop existing foreign key if present
|
||||||
|
if (Schema::hasColumn('pbg_task_payments', 'pbg_task_id')) {
|
||||||
|
$table->dropForeign(['pbg_task_id']);
|
||||||
|
// Make column nullable
|
||||||
|
$table->unsignedBigInteger('pbg_task_id')->nullable()->change();
|
||||||
|
// Recreate foreign key
|
||||||
|
$table->foreign('pbg_task_id')->references('id')->on('pbg_task')->cascadeOnDelete();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Drop legacy columns if no longer needed
|
||||||
|
if (Schema::hasColumn('pbg_task_payments', 'registration_number')) {
|
||||||
|
$table->dropIndex(['registration_number']);
|
||||||
|
$table->dropColumn('registration_number');
|
||||||
|
}
|
||||||
|
if (Schema::hasColumn('pbg_task_payments', 'sts_form_number')) {
|
||||||
|
$table->dropColumn('sts_form_number');
|
||||||
|
}
|
||||||
|
if (Schema::hasColumn('pbg_task_payments', 'payment_date')) {
|
||||||
|
$table->dropColumn('payment_date');
|
||||||
|
}
|
||||||
|
if (Schema::hasColumn('pbg_task_payments', 'pad_amount')) {
|
||||||
|
$table->dropColumn('pad_amount');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Make pbg_task_uid nullable
|
||||||
|
if (Schema::hasColumn('pbg_task_payments', 'pbg_task_uid')) {
|
||||||
|
$table->string('pbg_task_uid')->nullable()->change();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add new columns (renamed for table conventions)
|
||||||
|
$table->integer('row_no')->nullable();
|
||||||
|
$table->string('consultation_type')->nullable();
|
||||||
|
$table->string('source_registration_number')->nullable();
|
||||||
|
$table->string('owner_name')->nullable();
|
||||||
|
$table->text('building_location')->nullable();
|
||||||
|
$table->string('building_function')->nullable();
|
||||||
|
$table->string('building_name')->nullable();
|
||||||
|
$table->date('application_date_raw')->nullable();
|
||||||
|
$table->string('verification_status')->nullable();
|
||||||
|
$table->string('application_status')->nullable();
|
||||||
|
$table->text('owner_address')->nullable();
|
||||||
|
$table->string('owner_phone')->nullable();
|
||||||
|
$table->string('owner_email')->nullable();
|
||||||
|
$table->date('note_date_raw')->nullable();
|
||||||
|
$table->text('document_shortage_note')->nullable();
|
||||||
|
$table->string('image_url')->nullable();
|
||||||
|
$table->string('krk_kkpr')->nullable();
|
||||||
|
$table->string('krk_number')->nullable();
|
||||||
|
$table->string('lh')->nullable();
|
||||||
|
$table->string('ska')->nullable();
|
||||||
|
$table->string('remarks')->nullable();
|
||||||
|
$table->string('helpdesk')->nullable();
|
||||||
|
$table->string('person_in_charge')->nullable();
|
||||||
|
$table->string('pbg_operator')->nullable();
|
||||||
|
$table->string('ownership')->nullable();
|
||||||
|
$table->string('taru_potential')->nullable();
|
||||||
|
$table->string('agency_validation')->nullable();
|
||||||
|
$table->string('retribution_category')->nullable();
|
||||||
|
$table->string('ba_tpt_number')->nullable();
|
||||||
|
$table->date('ba_tpt_date_raw')->nullable();
|
||||||
|
$table->string('ba_tpa_number')->nullable();
|
||||||
|
$table->date('ba_tpa_date_raw')->nullable();
|
||||||
|
$table->string('skrd_number')->nullable();
|
||||||
|
$table->date('skrd_date_raw')->nullable();
|
||||||
|
$table->string('ptsp_status')->nullable();
|
||||||
|
$table->string('issued_status')->nullable();
|
||||||
|
$table->date('payment_date_raw')->nullable();
|
||||||
|
$table->string('sts_format')->nullable();
|
||||||
|
$table->integer('issuance_year')->nullable();
|
||||||
|
$table->integer('current_year')->nullable();
|
||||||
|
$table->string('village')->nullable();
|
||||||
|
$table->string('district')->nullable();
|
||||||
|
$table->decimal('building_area', 18, 2)->nullable()->default(0);
|
||||||
|
$table->decimal('building_height', 18, 2)->nullable()->default(0);
|
||||||
|
$table->integer('floor_count')->nullable();
|
||||||
|
$table->integer('unit_count')->nullable();
|
||||||
|
$table->decimal('proposed_retribution', 18, 2)->nullable()->default(0);
|
||||||
|
$table->decimal('retribution_total_simbg', 18, 2)->nullable()->default(0);
|
||||||
|
$table->decimal('retribution_total_pad', 18, 2)->nullable()->default(0);
|
||||||
|
$table->decimal('penalty_amount', 18, 2)->nullable()->default(0);
|
||||||
|
$table->string('business_category')->nullable();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reverse the migrations.
|
||||||
|
*/
|
||||||
|
public function down(): void
|
||||||
|
{
|
||||||
|
Schema::table('pbg_task_payments', function (Blueprint $table) {
|
||||||
|
if (Schema::hasColumn('pbg_task_payments', 'pbg_task_id')) {
|
||||||
|
// Drop FK, revert to not nullable, recreate FK
|
||||||
|
$table->dropForeign(['pbg_task_id']);
|
||||||
|
$table->unsignedBigInteger('pbg_task_id')->nullable(false)->change();
|
||||||
|
$table->foreign('pbg_task_id')->references('id')->on('pbg_task')->cascadeOnDelete();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Revert pbg_task_uid to not nullable
|
||||||
|
if (Schema::hasColumn('pbg_task_payments', 'pbg_task_uid')) {
|
||||||
|
$table->string('pbg_task_uid')->nullable(false)->change();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Drop the added columns
|
||||||
|
$columns = [
|
||||||
|
'row_no','consultation_type','source_registration_number','owner_name','building_location','building_function','building_name','application_date_raw',
|
||||||
|
'verification_status','application_status','owner_address','owner_phone','owner_email','note_date_raw','document_shortage_note',
|
||||||
|
'image_url','krk_kkpr','krk_number','lh','ska','remarks','helpdesk','person_in_charge','pbg_operator','ownership','taru_potential',
|
||||||
|
'agency_validation','retribution_category','ba_tpt_number','ba_tpt_date_raw','ba_tpa_number','ba_tpa_date_raw',
|
||||||
|
'skrd_number','skrd_date_raw','ptsp_status','issued_status','payment_date_raw','sts_format','issuance_year',
|
||||||
|
'current_year','village','district','building_area','building_height','floor_count','unit_count','proposed_retribution','retribution_total_simbg',
|
||||||
|
'retribution_total_pad','penalty_amount','business_category'
|
||||||
|
];
|
||||||
|
foreach ($columns as $col) {
|
||||||
|
if (Schema::hasColumn('pbg_task_payments', $col)) {
|
||||||
|
$table->dropColumn($col);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Restore legacy columns
|
||||||
|
if (!Schema::hasColumn('pbg_task_payments', 'registration_number')) {
|
||||||
|
$table->string('registration_number');
|
||||||
|
$table->index('registration_number');
|
||||||
|
}
|
||||||
|
if (!Schema::hasColumn('pbg_task_payments', 'sts_form_number')) {
|
||||||
|
$table->string('sts_form_number')->nullable();
|
||||||
|
}
|
||||||
|
if (!Schema::hasColumn('pbg_task_payments', 'payment_date')) {
|
||||||
|
$table->date('payment_date')->nullable();
|
||||||
|
}
|
||||||
|
if (!Schema::hasColumn('pbg_task_payments', 'pad_amount')) {
|
||||||
|
$table->decimal('pad_amount', 18, 2)->default(0);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
Reference in New Issue
Block a user