From 2e385f80cdc4a4485c1f834baacdfac82bd4c7b3 Mon Sep 17 00:00:00 2001
From: arifal <arifalhidayat10@gmail.com>
Date: Fri, 14 Mar 2025 19:10:28 +0700
Subject: [PATCH] fix optimize syncronize

---
 app/Console/Commands/ScrapingData.php |  86 ++++++++
 app/Services/ServiceGoogleSheet.php   | 284 ++++++++++++++++++++++++++
 app/Services/ServicePbgTask.php       |  62 ++++++
 app/Services/ServiceSIMBG.php         | 112 +++++-----
 app/Services/ServiceTokenSIMBG.php    |  79 +++++++
 5 files changed, 569 insertions(+), 54 deletions(-)
 create mode 100644 app/Console/Commands/ScrapingData.php
 create mode 100644 app/Services/ServiceGoogleSheet.php
 create mode 100644 app/Services/ServicePbgTask.php
 create mode 100644 app/Services/ServiceTokenSIMBG.php

diff --git a/app/Console/Commands/ScrapingData.php b/app/Console/Commands/ScrapingData.php
new file mode 100644
index 0000000..b9b3614
--- /dev/null
+++ b/app/Console/Commands/ScrapingData.php
@@ -0,0 +1,86 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Models\ImportDatasource;
+use App\Services\ServiceGoogleSheet;
+use App\Services\ServicePbgTask;
+use GuzzleHttp\Client; // Import Guzzle Client
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\DB;
+
+class ScrapingData extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'app:scraping-data';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Command description';
+
+    private $client;
+    private $service_pbg_task;
+
+    /**
+     * Inject dependencies.
+     */
+    public function __construct(Client $client, ServicePbgTask $service_pbg_task)
+    {
+        parent::__construct();
+        $this->client = $client;
+        $this->service_pbg_task = $service_pbg_task;
+    }
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        DB::beginTransaction(); // Start transaction
+
+        try {
+            // Create a record with "processing" status
+            $import_datasource = ImportDatasource::create([
+                'message' => 'Initiating scraping...',
+                'response_body' => null,
+                'status' => 'processing'
+            ]);
+
+            // Run the service
+            $service_google_sheet = new ServiceGoogleSheet($import_datasource->id);
+            $service_google_sheet->run_service();
+
+            // Run the ServicePbgTask with injected Guzzle Client
+            $this->service_pbg_task->run_service();
+
+            // Update the record status to "success" after completion
+            $import_datasource->update([
+                'status' => 'success',
+                'message' => 'Scraping completed successfully.'
+            ]);
+
+            DB::commit(); // Commit the transaction if everything is successful
+        } catch (\Exception $e) {
+            DB::rollBack(); // Rollback transaction on error
+
+            // Log the error for debugging
+            Log::error('Scraping failed: ' . $e->getMessage(), ['trace' => $e->getTraceAsString()]);
+
+            // Handle errors by updating the status to "failed"
+            if (isset($import_datasource)) {
+                $import_datasource->update([
+                    'status' => 'failed',
+                    'response_body' => 'Error: ' . $e->getMessage()
+                ]);
+            }
+        }
+    }
+}
diff --git a/app/Services/ServiceGoogleSheet.php b/app/Services/ServiceGoogleSheet.php
new file mode 100644
index 0000000..00d8653
--- /dev/null
+++ b/app/Services/ServiceGoogleSheet.php
@@ -0,0 +1,284 @@
+<?php
+
+namespace App\Services;
+use App\Models\DataSetting;
+use App\Models\ImportDatasource;
+use App\Models\PbgTaskGoogleSheet;
+use Carbon\Carbon;
+use Google_Client;
+use Google_Service_Sheets;
+use Log;
+class ServiceGoogleSheet
+{
+    protected $client;
+    protected $service;
+    protected $spreadsheetID;
+    protected $service_sheets;
+    protected $import_datasource;
+    public function __construct(int $import_datasource_id)
+    {
+        $this->client = new Google_Client();
+        $this->client->setApplicationName("Sibedas Google Sheets API");
+        $this->client->setScopes([Google_Service_Sheets::SPREADSHEETS_READONLY]);
+        $this->client->setAuthConfig(storage_path("app/teak-banner-450003-s8-ea05661d9db0.json"));
+        $this->client->setAccessType("offline");
+
+        $this->service = new Google_Service_Sheets($this->client);
+        $this->spreadsheetID = env("SPREAD_SHEET_ID");
+
+        $this->service_sheets = new Google_Service_Sheets($this->client);
+        $this->import_datasource = ImportDatasource::findOrFail($import_datasource_id);
+    }
+
+    public function run_service(){
+        $run_one = $this->sync_big_data();
+        if(!$run_one){
+            $this->import_datasource->update(['status' => 'failed']);
+            return false;
+        }
+        $run_two = $this->sync_google_sheet_data();
+        if(!$run_two){
+            $this->import_datasource->update(['status' => 'failed']);
+            return false;
+        }
+        return true;
+    }
+    public function sync_google_sheet_data() {
+        try {
+            $sheet_data = $this->get_data_by_sheet(0);
+
+            if (empty($sheet_data) || count($sheet_data) < 2) {
+                Log::warning("sync_google_sheet_data: No valid data found.");
+                return false;
+            }
+
+            $cleanValue = function ($value) {
+                return (isset($value) && trim($value) !== '') ? trim($value) : null;
+            };
+
+            $mapUpsert = [];
+            foreach(array_slice($sheet_data, 1) as $row){
+                if(!is_array($row)){
+                    continue;
+                }
+
+                $no_registrasi = $cleanValue($row[2] ?? null);
+
+                // Apply the same logic from your SQL UPDATE
+                if (strpos($no_registrasi, 'PBG-') === 0) {
+                    $format_registrasi = $no_registrasi;
+                } else {
+                    $format_registrasi = sprintf(
+                        "PBG-%s-%s-%s",
+                        substr($no_registrasi, 0, 6) ?: '',
+                        substr($no_registrasi, 7, 8) ?: '',
+                        substr($no_registrasi, -2) ?: ''
+                    );
+                }
+
+                $mapUpsert[] = [
+                    'jenis_konsultasi' => $cleanValue($row[1] ?? null),
+                    'no_registrasi' => $no_registrasi,
+                    'formatted_registration_number' => $format_registrasi,
+                    'nama_pemilik' => $cleanValue($row[3] ?? null),
+                    'lokasi_bg' => $cleanValue($row[4] ?? null),
+                    'fungsi_bg' => $cleanValue($row[5] ?? null),
+                    'nama_bangunan' => $cleanValue($row[6] ?? null),
+                    'tgl_permohonan' => $this->convertToDate($cleanValue($row[7] ?? null)),
+                    'status_verifikasi' => $cleanValue($row[8] ?? null),
+                    'status_permohonan' => $cleanValue($row[9] ?? null),
+                    'alamat_pemilik' => $cleanValue($row[10] ?? null),
+                    'no_hp' => $cleanValue($row[11] ?? null),
+                    'email' => $cleanValue($row[12] ?? null),
+                    'tanggal_catatan' => $this->convertToDate($cleanValue($row[13] ?? null)),
+                    'catatan_kekurangan_dokumen' => $cleanValue($row[14] ?? null),
+                    'gambar' => $cleanValue($row[15] ?? null),
+                    'krk_kkpr' => $cleanValue($row[16] ?? null),
+                    'no_krk' => $cleanValue($row[17] ?? null),
+                    'lh' => $cleanValue($row[18] ?? null),
+                    'ska' => $cleanValue($row[19] ?? null),
+                    'keterangan' => $cleanValue($row[20] ?? null),
+                    'helpdesk' => $cleanValue($row[21] ?? null),
+                    'pj' => $cleanValue($row[22] ?? null),
+                    'kepemilikan' => $cleanValue($row[24] ?? null),
+                    'potensi_taru' => $cleanValue($row[25] ?? null),
+                    'validasi_dinas' => $cleanValue($row[26] ?? null),
+                    'kategori_retribusi' => $cleanValue($row[27] ?? null),
+                    'no_urut_ba_tpt' => $cleanValue($row[28] ?? null),
+                    'tanggal_ba_tpt' => $this->convertToDate($cleanValue($row[29] ?? null)),
+                    'no_urut_ba_tpa' => $cleanValue($row[30] ?? null),
+                    'tanggal_ba_tpa' => $this->convertToDate($cleanValue($row[31] ?? null)),
+                    'no_urut_skrd' => $cleanValue($row[32] ?? null),
+                    'tanggal_skrd' => $this->convertToDate($cleanValue($row[33] ?? null)),
+                    'ptsp' => $cleanValue($row[34] ?? null),
+                    'selesai_terbit' => $cleanValue($row[35] ?? null),
+                    'tanggal_pembayaran' => $cleanValue($row[36] ?? null),
+                    'format_sts' => $cleanValue($row[37] ?? null),
+                    'tahun_terbit' => (int) $cleanValue($row[38] ?? null),
+                    'tahun_berjalan' => (int) $cleanValue($row[39] ?? null),
+                    'kelurahan' => $cleanValue($row[40] ?? null),
+                    'kecamatan' => $cleanValue($row[41] ?? null),
+                    'lb' => $this->convertToDecimal($cleanValue($row[42] ?? 0)),
+                    'tb' => $this->convertToDecimal($cleanValue($row[43] ?? 0)),
+                    'jlb' => (int) $cleanValue($row[44] ?? null),
+                    'unit' => (int) $cleanValue($row[45] ?? null),
+                    'usulan_retribusi' => (int) $cleanValue($row[46] ?? null),
+                    'nilai_retribusi_keseluruhan_simbg' => $this->convertToDecimal($cleanValue($row[47] ?? 0)),
+                    'nilai_retribusi_keseluruhan_pad' => $this->convertToDecimal($cleanValue($row[48] ?? 0)),
+                    'denda' => $this->convertToDecimal($cleanValue($row[49] ?? 0)),
+                    'latitude' => $cleanValue($row[50] ?? null),
+                    'longitude' => $cleanValue($row[51] ?? null),
+                    'nik_nib' => $cleanValue($row[52] ?? null),
+                    'dok_tanah' => $cleanValue($row[53] ?? null),
+                    'temuan' => $cleanValue($row[54] ?? null),
+                    'updated_at' => now()
+                ];
+            }
+
+            // Count occurrences of each no_registrasi
+            $registrasiCounts = array_count_values(array_column($mapUpsert, 'no_registrasi'));
+
+            // Filter duplicates (those appearing more than once)
+            $duplicates = array_filter($registrasiCounts, function ($count) {
+                return $count > 1;
+            });
+
+            if (!empty($duplicates)) {
+                Log::warning("Duplicate no_registrasi found", ['duplicates' => array_keys($duplicates)]);
+            }
+
+            // Remove duplicates before upsert
+            $mapUpsert = collect($mapUpsert)->unique('no_registrasi')->values()->all();
+
+            $batchSize = 1000;
+            $chunks = array_chunk($mapUpsert, $batchSize);
+            foreach ($chunks as $chunk) {
+                PbgTaskGoogleSheet::upsert($chunk, ['no_registrasi']);
+            }
+
+            Log::info("sync google sheet done");
+        } catch (\Exception $e) {
+            Log::error("sync_google_sheet_data failed", ['error' => $e->getMessage()]);
+            return false;
+        }
+    }
+
+
+    public function sync_big_data(){
+        try {
+            $sheet_big_data = $this->get_data_by_sheet();
+            $data_setting_result = []; // Initialize result storage
+
+            $found_section = null; // Track which section is found
+
+            foreach ($sheet_big_data as $row) {
+                // Check for section headers
+                if (in_array("•PROSES PENERBITAN:", $row)) {
+                    $found_section = "MENUNGGU_KLIK_DPMPTSP";
+                } elseif (in_array("•BERKAS AKTUAL TERVERIFIKASI DINAS TEKNIS 2024:", $row)) {
+                    $found_section = "REALISASI_TERBIT_PBG";
+                } elseif (in_array("•TERPROSES DI DPUTR: belum selesai rekomtek'", $row)) {
+                    $found_section = "PROSES_DINAS_TEKNIS";
+                }
+
+                // If a section is found and we reach "Grand Total", save the corresponding values
+                if ($found_section && isset($row[0]) && trim($row[0]) === "Grand Total") {
+                    if ($found_section === "MENUNGGU_KLIK_DPMPTSP") {
+                        $data_setting_result["MENUNGGU_KLIK_DPMPTSP_COUNT"] = $this->convertToInteger($row[2]) ?? null;
+                        $data_setting_result["MENUNGGU_KLIK_DPMPTSP_SUM"] = $this->convertToDecimal($row[3])  ?? null;
+                    } elseif ($found_section === "REALISASI_TERBIT_PBG") {
+                        $data_setting_result["REALISASI_TERBIT_PBG_COUNT"] = $this->convertToInteger($row[2]) ?? null;
+                        $data_setting_result["REALISASI_TERBIT_PBG_SUM"] = $this->convertToDecimal($row[4]) ?? null;
+                    } elseif ($found_section === "PROSES_DINAS_TEKNIS") {
+                        $data_setting_result["PROSES_DINAS_TEKNIS_COUNT"] = $this->convertToInteger($row[2]) ?? null;
+                        $data_setting_result["PROSES_DINAS_TEKNIS_SUM"] = $this->convertToDecimal($row[3]) ?? null;
+                    }
+
+                    // Reset section tracking after capturing "Grand Total"
+                    $found_section = null;
+                }
+            }
+
+            foreach ($data_setting_result as $key => $value) {
+                DataSetting::updateOrInsert(
+                    ["key" => $key], // Find by key
+                    ["value" => $value] // Update or insert value
+                );
+            }
+
+            return true;
+        } catch (\Exception $e) {
+            // **Log error**
+            Log::error("Error syncing Google Sheet data", ['error' => $e->getMessage()]);
+            return false;
+        }
+    }
+
+    private function get_data_by_sheet($no_sheet = 1){
+        $spreadsheet = $this->service->spreadsheets->get($this->spreadsheetID);
+        $sheets = $spreadsheet->getSheets();
+        $sheetTitle = $sheets[$no_sheet]->getProperties()->getTitle();
+        $range = "{$sheetTitle}";
+        $response = $this->service->spreadsheets_values->get($this->spreadsheetID, $range);
+        $values = $response->getValues();
+        return!empty($values)? $values : [];
+    }
+
+    private function convertToInteger($value) {
+        // Check if the value is an empty string, and return null if true
+        if (trim($value) === "") {
+            return null;
+        }
+
+        $cleaned = str_replace('.','', $value);
+
+        // Otherwise, cast to integer
+        return (int) $cleaned;
+    }
+
+    private function convertToDecimal(?string $value): ?float
+    {
+        if (empty($value)) {
+            return null; // Return null if the input is empty
+        }
+
+        // Remove all non-numeric characters except comma and dot
+        $value = preg_replace('/[^0-9,\.]/', '', $value);
+
+        // If the number contains both dot (.) and comma (,)
+        if (strpos($value, '.') !== false && strpos($value, ',') !== false) {
+            $value = str_replace('.', '', $value); // Remove thousands separator
+            $value = str_replace(',', '.', $value); // Convert decimal separator to dot
+        }
+        // If only a dot is present (assumed as thousands separator)
+        elseif (strpos($value, '.') !== false) {
+            $value = str_replace('.', '', $value); // Remove all dots (treat as thousands separators)
+        }
+        // If only a comma is present (assumed as decimal separator)
+        elseif (strpos($value, ',') !== false) {
+            $value = str_replace(',', '.', $value); // Convert comma to dot (decimal separator)
+        }
+
+        // Ensure the value is numeric before returning
+        return is_numeric($value) ? (float) number_format((float) $value, 2, '.', '') : null;
+    }
+
+    private function convertToDate($dateString)
+    {
+        try {
+            // Check if the string is empty
+            if (empty($dateString)) {
+                return null;
+            }
+
+            // Try to parse the date string
+            $date = Carbon::parse($dateString);
+
+            // Return the Carbon instance
+            return $date->format('Y-m-d');
+        } catch (\Exception $e) {
+            // Return null if an error occurs during parsing
+            return null;
+        }
+    }
+}
diff --git a/app/Services/ServicePbgTask.php b/app/Services/ServicePbgTask.php
new file mode 100644
index 0000000..2c0e513
--- /dev/null
+++ b/app/Services/ServicePbgTask.php
@@ -0,0 +1,62 @@
+<?php
+
+namespace App\Services;
+
+use App\Models\GlobalSetting;
+use GuzzleHttp\Client;
+use Exception;
+use Illuminate\Support\Facades\Log;
+
+class ServicePbgTask
+{
+    private $client;
+    private $simbg_host;
+    private $fetch_per_page;
+    private $pbg_task_url;
+    private $service_token;
+
+    public function __construct(Client $client, ServiceTokenSIMBG $service_token)
+    {
+        $settings = GlobalSetting::whereIn('key', ['SIMBG_HOST', 'FETCH_PER_PAGE'])
+            ->pluck('value', 'key');
+
+        $this->simbg_host = trim((string) ($settings['SIMBG_HOST'] ?? ""));
+        // $this->fetch_per_page = trim((string) ($settings['FETCH_PER_PAGE'] ?? "10"));
+        $this->fetch_per_page = 10;
+        $this->client = $client;
+        $this->service_token = $service_token;
+        $this->pbg_task_url = "{$this->simbg_host}/api/pbg/v1/list/?page=1&size={$this->fetch_per_page}&sort=ASC";
+    }
+
+    public function run_service()
+    {
+        $this->fetch_pbg_task();
+    }
+
+    private function fetch_pbg_task()
+    {
+        try {
+            $token = $this->service_token->get_token();
+            $options = [
+                'headers' => [
+                    'Authorization' => "Bearer {$token['access']}",
+                    'Accept'        => 'application/json'
+                ]
+            ];
+
+            $fetch_data = $this->client->get($this->pbg_task_url, $options);
+            $response_status_code = $fetch_data->getStatusCode();
+            $response = json_decode($fetch_data->getBody()->getContents(), true);
+            $data = $response['data'];
+            $total_page = $response['total_page'];
+
+            Log::info("Successfully fetched PBG tasks", ['data' => $data]);
+            Log::info("Status code response", ['code' => $response_status_code]);
+            Log::info("Success", ['total_page' => $total_page]);
+            return $response;
+        } catch (Exception $e) {
+            Log::error("Failed to fetch PBG tasks", ['error' => $e->getMessage()]);
+            return null; // Return null if an error occurs
+        }
+    }
+}
diff --git a/app/Services/ServiceSIMBG.php b/app/Services/ServiceSIMBG.php
index 773a83b..466ec3a 100644
--- a/app/Services/ServiceSIMBG.php
+++ b/app/Services/ServiceSIMBG.php
@@ -179,60 +179,59 @@ class ServiceSIMBG
             }
             $mapToUpsert = [];
 
-            foreach($sheetData as $data){
-                $mapToUpsert[] = 
-                    [
-                        'no_registrasi' => $data['no__registrasi'] ?? null,
-                        'jenis_konsultasi' => $data['jenis_konsultasi'] ?? null,
-                        'fungsi_bg' => $data['fungsi_bg'] ?? null,
-                        'tgl_permohonan' => $this->convertToDate($data['tgl_permohonan']),
-                        'status_verifikasi' => $data['status_verifikasi'] ?? null,
-                        'status_permohonan' => $this->convertToDate($data['status_permohonan']),
-                        'alamat_pemilik' => $data['alamat_pemilik'] ?? null,
-                        'no_hp' => $data['no__hp'] ?? null,
-                        'email' => $data['e_mail'] ?? null,
-                        'tanggal_catatan' => $this->convertToDate($data['tanggal_catatan']),
-                        'catatan_kekurangan_dokumen' => $data['catatan_kekurangan_dokumen'] ?? null,
-                        'gambar' => $data['gambar'] ?? null,
-                        'krk_kkpr' => $data['krk_kkpr'] ?? null,
-                        'no_krk' => $data['no__krk'] ?? null,
-                        'lh' => $data['lh'] ?? null,
-                        'ska' => $data['ska'] ?? null,
-                        'keterangan' => $data['keterangan'] ?? null,
-                        'helpdesk' => $data['helpdesk'] ?? null,
-                        'pj' => $data['pj'] ?? null,
-                        'kepemilikan' => $data['kepemilikan'] ?? null,
-                        'potensi_taru' => $data['potensi_taru'] ?? null,
-                        'validasi_dinas' => $data['validasi_dinas'] ?? null,
-                        'kategori_retribusi' => $data['kategori_retribusi'] ?? null,
-                        'no_urut_ba_tpt' => $data['no__urut_ba_tpt__2024_0001_'] ?? null,
-                        'tanggal_ba_tpt' => $this->convertToDate($data['tanggal_ba_tpt']),
-                        'no_urut_ba_tpa' => $data['no__urut_ba_tpa'] ?? null,
-                        'tanggal_ba_tpa' => $this->convertToDate($data['tanggal_ba_tpa']),
-                        'no_urut_skrd' => $data['no__urut_skrd__2024_0001_'] ?? null,
-                        'tanggal_skrd' => $this->convertToDate($data['tanggal_skrd']),
-                        'ptsp' => $data['ptsp'] ?? null,
-                        'selesai_terbit' => $data['selesai_terbit'] ?? null,
-                        'tanggal_pembayaran' => $this->convertToDate($data['tanggal_pembayaran__yyyy_mm_dd_']),
-                        'format_sts' => $data['format_sts'] ?? null,
-                        'tahun_terbit' => (int) $data['tahun_terbit'] ?? null,
-                        'tahun_berjalan' => (int) $data['tahun_berjalan'] ?? null,
-                        'kelurahan' => $data['kelurahan'] ?? null,
-                        'kecamatan' => $data['kecamatan'] ?? null,
-                        'lb' => $this->convertToDecimal($data['lb'])  ?? null,
-                        'tb' => $this->convertToDecimal($data['tb']) ?? null,
-                        'jlb' => (int) $data['jlb'] ?? null,
-                        'unit' => (int) $data['unit'] ?? null,
-                        'usulan_retribusi' => (int) $data['usulan_retribusi'] ?? null,
-                        'nilai_retribusi_keseluruhan_simbg' => $this->convertToDecimal($data['nilai_retribusi_keseluruhan__simbg_']) ?? null,
-                        'nilai_retribusi_keseluruhan_pad' => $this->convertToDecimal($data['nilai_retribusi_keseluruhan__pad_']) ?? null,
-                        'denda' => $this->convertToDecimal($data['denda']) ?? null,
-                        'latitude' => $data['latitude'] ?? null,
-                        'longitude' => $data['longitude'] ?? null,
-                        'nik_nib' => $data['nik_nib'] ?? null,
-                        'dok_tanah' => $data['dok__tanah'] ?? null,
-                        'temuan' => $data['temuan'] ?? null,
-                    ];
+            foreach ($sheetData as $data) {
+                $mapToUpsert[] = [
+                    'no_registrasi' => $this->cleanString($data['no__registrasi'] ?? null),
+                    'jenis_konsultasi' => $this->cleanString($data['jenis_konsultasi'] ?? null),
+                    'fungsi_bg' => $this->cleanString($data['fungsi_bg'] ?? null),
+                    'tgl_permohonan' => $this->convertToDate($this->cleanString($data['tgl_permohonan'] ?? null)),
+                    'status_verifikasi' => $this->cleanString($data['status_verifikasi'] ?? null),
+                    'status_permohonan' => $this->convertToDate($this->cleanString($data['status_permohonan'] ?? null)),
+                    'alamat_pemilik' => $this->cleanString($data['alamat_pemilik'] ?? null),
+                    'no_hp' => $this->cleanString($data['no__hp'] ?? null),
+                    'email' => $this->cleanString($data['e_mail'] ?? null),
+                    'tanggal_catatan' => $this->convertToDate($this->cleanString($data['tanggal_catatan'] ?? null)),
+                    'catatan_kekurangan_dokumen' => $this->cleanString($data['catatan_kekurangan_dokumen'] ?? null),
+                    'gambar' => $this->cleanString($data['gambar'] ?? null),
+                    'krk_kkpr' => $this->cleanString($data['krk_kkpr'] ?? null),
+                    'no_krk' => $this->cleanString($data['no__krk'] ?? null),
+                    'lh' => $this->cleanString($data['lh'] ?? null),
+                    'ska' => $this->cleanString($data['ska'] ?? null),
+                    'keterangan' => $this->cleanString($data['keterangan'] ?? null),
+                    'helpdesk' => $this->cleanString($data['helpdesk'] ?? null),
+                    'pj' => $this->cleanString($data['pj'] ?? null),
+                    'kepemilikan' => $this->cleanString($data['kepemilikan'] ?? null),
+                    'potensi_taru' => $this->cleanString($data['potensi_taru'] ?? null),
+                    'validasi_dinas' => $this->cleanString($data['validasi_dinas'] ?? null),
+                    'kategori_retribusi' => $this->cleanString($data['kategori_retribusi'] ?? null),
+                    'no_urut_ba_tpt' => $this->cleanString($data['no__urut_ba_tpt__2024_0001_'] ?? null),
+                    'tanggal_ba_tpt' => $this->convertToDate($this->cleanString($data['tanggal_ba_tpt'] ?? null)),
+                    'no_urut_ba_tpa' => $this->cleanString($data['no__urut_ba_tpa'] ?? null),
+                    'tanggal_ba_tpa' => $this->convertToDate($this->cleanString($data['tanggal_ba_tpa'] ?? null)),
+                    'no_urut_skrd' => $this->cleanString($data['no__urut_skrd__2024_0001_'] ?? null),
+                    'tanggal_skrd' => $this->convertToDate($this->cleanString($data['tanggal_skrd'] ?? null)),
+                    'ptsp' => $this->cleanString($data['ptsp'] ?? null),
+                    'selesai_terbit' => $this->cleanString($data['selesai_terbit'] ?? null),
+                    'tanggal_pembayaran' => $this->convertToDate($this->cleanString($data['tanggal_pembayaran__yyyy_mm_dd_'] ?? null)),
+                    'format_sts' => $this->cleanString($data['format_sts'] ?? null),
+                    'tahun_terbit' => (int) ($data['tahun_terbit'] ?? null),
+                    'tahun_berjalan' => (int) ($data['tahun_berjalan'] ?? null),
+                    'kelurahan' => $this->cleanString($data['kelurahan'] ?? null),
+                    'kecamatan' => $this->cleanString($data['kecamatan'] ?? null),
+                    'lb' => $this->convertToDecimal($data['lb'] ?? null),
+                    'tb' => $this->convertToDecimal($data['tb'] ?? null),
+                    'jlb' => (int) ($data['jlb'] ?? null),
+                    'unit' => (int) ($data['unit'] ?? null),
+                    'usulan_retribusi' => (int) ($data['usulan_retribusi'] ?? null),
+                    'nilai_retribusi_keseluruhan_simbg' => $this->convertToDecimal($data['nilai_retribusi_keseluruhan__simbg_'] ?? null),
+                    'nilai_retribusi_keseluruhan_pad' => $this->convertToDecimal($data['nilai_retribusi_keseluruhan__pad_'] ?? null),
+                    'denda' => $this->convertToDecimal($data['denda'] ?? null),
+                    'latitude' => $this->cleanString($data['latitude'] ?? null),
+                    'longitude' => $this->cleanString($data['longitude'] ?? null),
+                    'nik_nib' => $this->cleanString($data['nik_nib'] ?? null),
+                    'dok_tanah' => $this->cleanString($data['dok__tanah'] ?? null),
+                    'temuan' => $this->cleanString($data['temuan'] ?? null),
+                ];
             }
 
             $batchSize = 1000;
@@ -656,4 +655,9 @@ class ServiceSIMBG
             return null;
         }
     }
+
+    private function cleanString($value)
+    {
+        return isset($value) ? trim(strip_tags($value)) : null;
+    }
 }
\ No newline at end of file
diff --git a/app/Services/ServiceTokenSIMBG.php b/app/Services/ServiceTokenSIMBG.php
new file mode 100644
index 0000000..2279b69
--- /dev/null
+++ b/app/Services/ServiceTokenSIMBG.php
@@ -0,0 +1,79 @@
+<?php
+
+namespace App\Services;
+use App\Models\GlobalSetting;
+use GuzzleHttp\Client;
+use GuzzleHttp\Exception\RequestException;
+use Illuminate\Support\Facades\Log;
+class ServiceTokenSIMBG
+{
+    private $client;
+    private $login_url;
+    private $email;
+    private $password;
+    private $simbg_host;
+    private $fetch_per_page;
+    private $refresh_url;
+
+    public function __construct()
+    {
+        $settings = GlobalSetting::whereIn('key', [
+            'SIMBG_EMAIL', 'SIMBG_PASSWORD', 'SIMBG_HOST', 'FETCH_PER_PAGE'
+        ])->pluck('value', 'key');
+        $this->email = trim((string) ($settings['SIMBG_EMAIL'] ?? ""));
+        $this->password = trim((string) ($settings['SIMBG_PASSWORD'] ?? ""));
+        $this->simbg_host = trim((string) ($settings['SIMBG_HOST'] ?? ""));
+        $this->fetch_per_page = trim((string) ($settings['FETCH_PER_PAGE'] ?? ""));
+        $this->client = new Client();
+        $this->login_url = $this->simbg_host . "/api/user/v1/auth/login/";
+        $this->refresh_url = $this->simbg_host. "/api/user/v1/auth/token/refresh/";
+    }
+
+    public function get_token(){
+        try {
+            $response = $this->client->request('POST', $this->login_url, [
+                'headers' => [
+                    'Content-Type' => 'application/json',
+                    'Accept' => 'application/json',
+                ],
+                'json' => [
+                    'email' => $this->email,
+                    'password' => $this->password
+                ]
+            ]);
+
+            $data = json_decode($response->getBody()->getContents(), true);
+
+            return $data['token'];
+        } catch (RequestException $e) {
+            Log::error("Failed to get token", [
+                'error' => $e->getMessage(),
+                'response' => $e->getResponse() ? $e->getResponse()->getBody()->getContents() : null
+            ]);
+            return null;
+        }
+    }
+
+    public function refresh_token(string $refresh_token){
+        try {
+            $response = $this->client->request('POST', $this->refresh_url, [
+                'headers' => [
+                    'Content-Type' => 'application/json',
+                    'Accept' => 'application/json',
+                ],
+                'json' => [
+                    'refresh' => $refresh_token
+                ]
+            ]);
+
+            $data = json_decode($response->getBody()->getContents(), true);
+
+            return $data;
+        } catch (\Throwable $th) {
+            Log::error("Failed to refresh token", [
+                'error' => $th->getMessage()
+            ]);
+            return null;
+        }
+    }
+}