arifal/sibedas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix optimize syncronize

Browse Source
This commit is contained in:
arifal
2025-03-14 19:10:28 +07:00
parent e2c26e0eff
commit 2e385f80cd
5 changed files with 569 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

284
app/Services/ServiceGoogleSheet.php Normal file
View File

@@ -0,0 +1,284 @@
<?php
namespace App\Services;
use App\Models\DataSetting;
use App\Models\ImportDatasource;
use App\Models\PbgTaskGoogleSheet;
use Carbon\Carbon;
use Google_Client;
use Google_Service_Sheets;
use Log;
class ServiceGoogleSheet
{
protected $client;
protected $service;
protected $spreadsheetID;
protected $service_sheets;
protected $import_datasource;
public function __construct(int $import_datasource_id)
{
$this->client = new Google_Client();
$this->client->setApplicationName("Sibedas Google Sheets API");
$this->client->setScopes([Google_Service_Sheets::SPREADSHEETS_READONLY]);
$this->client->setAuthConfig(storage_path("app/teak-banner-450003-s8-ea05661d9db0.json"));
$this->client->setAccessType("offline");
$this->service = new Google_Service_Sheets($this->client);
$this->spreadsheetID = env("SPREAD_SHEET_ID");
$this->service_sheets = new Google_Service_Sheets($this->client);
$this->import_datasource = ImportDatasource::findOrFail($import_datasource_id);
}
public function run_service(){
$run_one = $this->sync_big_data();
if(!$run_one){
$this->import_datasource->update(['status' => 'failed']);
return false;
}
$run_two = $this->sync_google_sheet_data();
if(!$run_two){
$this->import_datasource->update(['status' => 'failed']);
return false;
}
return true;
}
public function sync_google_sheet_data() {
try {
$sheet_data = $this->get_data_by_sheet(0);
if (empty($sheet_data) || count($sheet_data) < 2) {
Log::warning("sync_google_sheet_data: No valid data found.");
return false;
}
$cleanValue = function ($value) {
return (isset($value) && trim($value) !== '') ? trim($value) : null;
};
$mapUpsert = [];
foreach(array_slice($sheet_data, 1) as $row){
if(!is_array($row)){
continue;
}
$no_registrasi = $cleanValue($row[2] ?? null);
// Apply the same logic from your SQL UPDATE
if (strpos($no_registrasi, 'PBG-') === 0) {
$format_registrasi = $no_registrasi;
} else {
$format_registrasi = sprintf(
"PBG-%s-%s-%s",
substr($no_registrasi, 0, 6) ?: '',
substr($no_registrasi, 7, 8) ?: '',
substr($no_registrasi, -2) ?: ''
);
}
$mapUpsert[] = [
'jenis_konsultasi' => $cleanValue($row[1] ?? null),
'no_registrasi' => $no_registrasi,
'formatted_registration_number' => $format_registrasi,
'nama_pemilik' => $cleanValue($row[3] ?? null),
'lokasi_bg' => $cleanValue($row[4] ?? null),
'fungsi_bg' => $cleanValue($row[5] ?? null),
'nama_bangunan' => $cleanValue($row[6] ?? null),
'tgl_permohonan' => $this->convertToDate($cleanValue($row[7] ?? null)),
'status_verifikasi' => $cleanValue($row[8] ?? null),
'status_permohonan' => $cleanValue($row[9] ?? null),
'alamat_pemilik' => $cleanValue($row[10] ?? null),
'no_hp' => $cleanValue($row[11] ?? null),
'email' => $cleanValue($row[12] ?? null),
'tanggal_catatan' => $this->convertToDate($cleanValue($row[13] ?? null)),
'catatan_kekurangan_dokumen' => $cleanValue($row[14] ?? null),
'gambar' => $cleanValue($row[15] ?? null),
'krk_kkpr' => $cleanValue($row[16] ?? null),
'no_krk' => $cleanValue($row[17] ?? null),
'lh' => $cleanValue($row[18] ?? null),
'ska' => $cleanValue($row[19] ?? null),
'keterangan' => $cleanValue($row[20] ?? null),
'helpdesk' => $cleanValue($row[21] ?? null),
'pj' => $cleanValue($row[22] ?? null),
'kepemilikan' => $cleanValue($row[24] ?? null),
'potensi_taru' => $cleanValue($row[25] ?? null),
'validasi_dinas' => $cleanValue($row[26] ?? null),
'kategori_retribusi' => $cleanValue($row[27] ?? null),
'no_urut_ba_tpt' => $cleanValue($row[28] ?? null),
'tanggal_ba_tpt' => $this->convertToDate($cleanValue($row[29] ?? null)),
'no_urut_ba_tpa' => $cleanValue($row[30] ?? null),
'tanggal_ba_tpa' => $this->convertToDate($cleanValue($row[31] ?? null)),
'no_urut_skrd' => $cleanValue($row[32] ?? null),
'tanggal_skrd' => $this->convertToDate($cleanValue($row[33] ?? null)),
'ptsp' => $cleanValue($row[34] ?? null),
'selesai_terbit' => $cleanValue($row[35] ?? null),
'tanggal_pembayaran' => $cleanValue($row[36] ?? null),
'format_sts' => $cleanValue($row[37] ?? null),
'tahun_terbit' => (int) $cleanValue($row[38] ?? null),
'tahun_berjalan' => (int) $cleanValue($row[39] ?? null),
'kelurahan' => $cleanValue($row[40] ?? null),
'kecamatan' => $cleanValue($row[41] ?? null),
'lb' => $this->convertToDecimal($cleanValue($row[42] ?? 0)),
'tb' => $this->convertToDecimal($cleanValue($row[43] ?? 0)),
'jlb' => (int) $cleanValue($row[44] ?? null),
'unit' => (int) $cleanValue($row[45] ?? null),
'usulan_retribusi' => (int) $cleanValue($row[46] ?? null),
'nilai_retribusi_keseluruhan_simbg' => $this->convertToDecimal($cleanValue($row[47] ?? 0)),
'nilai_retribusi_keseluruhan_pad' => $this->convertToDecimal($cleanValue($row[48] ?? 0)),
'denda' => $this->convertToDecimal($cleanValue($row[49] ?? 0)),
'latitude' => $cleanValue($row[50] ?? null),
'longitude' => $cleanValue($row[51] ?? null),
'nik_nib' => $cleanValue($row[52] ?? null),
'dok_tanah' => $cleanValue($row[53] ?? null),
'temuan' => $cleanValue($row[54] ?? null),
'updated_at' => now()
];
}
// Count occurrences of each no_registrasi
$registrasiCounts = array_count_values(array_column($mapUpsert, 'no_registrasi'));
// Filter duplicates (those appearing more than once)
$duplicates = array_filter($registrasiCounts, function ($count) {
return $count > 1;
});
if (!empty($duplicates)) {
Log::warning("Duplicate no_registrasi found", ['duplicates' => array_keys($duplicates)]);
}
// Remove duplicates before upsert
$mapUpsert = collect($mapUpsert)->unique('no_registrasi')->values()->all();
$batchSize = 1000;
$chunks = array_chunk($mapUpsert, $batchSize);
foreach ($chunks as $chunk) {
PbgTaskGoogleSheet::upsert($chunk, ['no_registrasi']);
}
Log::info("sync google sheet done");
} catch (\Exception $e) {
Log::error("sync_google_sheet_data failed", ['error' => $e->getMessage()]);
return false;
}
}
public function sync_big_data(){
try {
$sheet_big_data = $this->get_data_by_sheet();
$data_setting_result = []; // Initialize result storage
$found_section = null; // Track which section is found
foreach ($sheet_big_data as $row) {
// Check for section headers
if (in_array("•PROSES PENERBITAN:", $row)) {
$found_section = "MENUNGGU_KLIK_DPMPTSP";
} elseif (in_array("•BERKAS AKTUAL TERVERIFIKASI DINAS TEKNIS 2024:", $row)) {
$found_section = "REALISASI_TERBIT_PBG";
} elseif (in_array("•TERPROSES DI DPUTR: belum selesai rekomtek'", $row)) {
$found_section = "PROSES_DINAS_TEKNIS";
}
// If a section is found and we reach "Grand Total", save the corresponding values
if ($found_section && isset($row[0]) && trim($row[0]) === "Grand Total") {
if ($found_section === "MENUNGGU_KLIK_DPMPTSP") {
$data_setting_result["MENUNGGU_KLIK_DPMPTSP_COUNT"] = $this->convertToInteger($row[2]) ?? null;
$data_setting_result["MENUNGGU_KLIK_DPMPTSP_SUM"] = $this->convertToDecimal($row[3]) ?? null;
} elseif ($found_section === "REALISASI_TERBIT_PBG") {
$data_setting_result["REALISASI_TERBIT_PBG_COUNT"] = $this->convertToInteger($row[2]) ?? null;
$data_setting_result["REALISASI_TERBIT_PBG_SUM"] = $this->convertToDecimal($row[4]) ?? null;
} elseif ($found_section === "PROSES_DINAS_TEKNIS") {
$data_setting_result["PROSES_DINAS_TEKNIS_COUNT"] = $this->convertToInteger($row[2]) ?? null;
$data_setting_result["PROSES_DINAS_TEKNIS_SUM"] = $this->convertToDecimal($row[3]) ?? null;
}
// Reset section tracking after capturing "Grand Total"
$found_section = null;
}
}
foreach ($data_setting_result as $key => $value) {
DataSetting::updateOrInsert(
["key" => $key], // Find by key
["value" => $value] // Update or insert value
);
}
return true;
} catch (\Exception $e) {
// **Log error**
Log::error("Error syncing Google Sheet data", ['error' => $e->getMessage()]);
return false;
}
}
private function get_data_by_sheet($no_sheet = 1){
$spreadsheet = $this->service->spreadsheets->get($this->spreadsheetID);
$sheets = $spreadsheet->getSheets();
$sheetTitle = $sheets[$no_sheet]->getProperties()->getTitle();
$range = "{$sheetTitle}";
$response = $this->service->spreadsheets_values->get($this->spreadsheetID, $range);
$values = $response->getValues();
return!empty($values)? $values : [];
}
private function convertToInteger($value) {
// Check if the value is an empty string, and return null if true
if (trim($value) === "") {
return null;
}
$cleaned = str_replace('.','', $value);
// Otherwise, cast to integer
return (int) $cleaned;
}
private function convertToDecimal(?string $value): ?float
{
if (empty($value)) {
return null; // Return null if the input is empty
}
// Remove all non-numeric characters except comma and dot
$value = preg_replace('/[^0-9,\.]/', '', $value);
// If the number contains both dot (.) and comma (,)
if (strpos($value, '.') !== false && strpos($value, ',') !== false) {
$value = str_replace('.', '', $value); // Remove thousands separator
$value = str_replace(',', '.', $value); // Convert decimal separator to dot
}
// If only a dot is present (assumed as thousands separator)
elseif (strpos($value, '.') !== false) {
$value = str_replace('.', '', $value); // Remove all dots (treat as thousands separators)
}
// If only a comma is present (assumed as decimal separator)
elseif (strpos($value, ',') !== false) {
$value = str_replace(',', '.', $value); // Convert comma to dot (decimal separator)
}
// Ensure the value is numeric before returning
return is_numeric($value) ? (float) number_format((float) $value, 2, '.', '') : null;
}
private function convertToDate($dateString)
{
try {
// Check if the string is empty
if (empty($dateString)) {
return null;
}
// Try to parse the date string
$date = Carbon::parse($dateString);
// Return the Carbon instance
return $date->format('Y-m-d');
} catch (\Exception $e) {
// Return null if an error occurs during parsing
return null;
}
}
}

62
app/Services/ServicePbgTask.php Normal file
View File

@@ -0,0 +1,62 @@
<?php
namespace App\Services;
use App\Models\GlobalSetting;
use GuzzleHttp\Client;
use Exception;
use Illuminate\Support\Facades\Log;
class ServicePbgTask
{
private $client;
private $simbg_host;
private $fetch_per_page;
private $pbg_task_url;
private $service_token;
public function __construct(Client $client, ServiceTokenSIMBG $service_token)
{
$settings = GlobalSetting::whereIn('key', ['SIMBG_HOST', 'FETCH_PER_PAGE'])
->pluck('value', 'key');
$this->simbg_host = trim((string) ($settings['SIMBG_HOST'] ?? ""));
// $this->fetch_per_page = trim((string) ($settings['FETCH_PER_PAGE'] ?? "10"));
$this->fetch_per_page = 10;
$this->client = $client;
$this->service_token = $service_token;
$this->pbg_task_url = "{$this->simbg_host}/api/pbg/v1/list/?page=1&size={$this->fetch_per_page}&sort=ASC";
}
public function run_service()
{
$this->fetch_pbg_task();
}
private function fetch_pbg_task()
{
try {
$token = $this->service_token->get_token();
$options = [
'headers' => [
'Authorization' => "Bearer {$token['access']}",
'Accept' => 'application/json'
]
];
$fetch_data = $this->client->get($this->pbg_task_url, $options);
$response_status_code = $fetch_data->getStatusCode();
$response = json_decode($fetch_data->getBody()->getContents(), true);
$data = $response['data'];
$total_page = $response['total_page'];
Log::info("Successfully fetched PBG tasks", ['data' => $data]);
Log::info("Status code response", ['code' => $response_status_code]);
Log::info("Success", ['total_page' => $total_page]);
return $response;
} catch (Exception $e) {
Log::error("Failed to fetch PBG tasks", ['error' => $e->getMessage()]);
return null; // Return null if an error occurs
}
}
}

112
app/Services/ServiceSIMBG.php
View File

@@ -179,60 +179,59 @@ class ServiceSIMBG
}
$mapToUpsert = [];
foreach($sheetData as $data){
$mapToUpsert[] =
[
'no_registrasi' => $data['no__registrasi'] ?? null,
'jenis_konsultasi' => $data['jenis_konsultasi'] ?? null,
'fungsi_bg' => $data['fungsi_bg'] ?? null,
'tgl_permohonan' => $this->convertToDate($data['tgl_permohonan']),
'status_verifikasi' => $data['status_verifikasi'] ?? null,
'status_permohonan' => $this->convertToDate($data['status_permohonan']),
'alamat_pemilik' => $data['alamat_pemilik'] ?? null,
'no_hp' => $data['no__hp'] ?? null,
'email' => $data['e_mail'] ?? null,
'tanggal_catatan' => $this->convertToDate($data['tanggal_catatan']),
'catatan_kekurangan_dokumen' => $data['catatan_kekurangan_dokumen'] ?? null,
'gambar' => $data['gambar'] ?? null,
'krk_kkpr' => $data['krk_kkpr'] ?? null,
'no_krk' => $data['no__krk'] ?? null,
'lh' => $data['lh'] ?? null,
'ska' => $data['ska'] ?? null,
'keterangan' => $data['keterangan'] ?? null,
'helpdesk' => $data['helpdesk'] ?? null,
'pj' => $data['pj'] ?? null,
'kepemilikan' => $data['kepemilikan'] ?? null,
'potensi_taru' => $data['potensi_taru'] ?? null,
'validasi_dinas' => $data['validasi_dinas'] ?? null,
'kategori_retribusi' => $data['kategori_retribusi'] ?? null,
'no_urut_ba_tpt' => $data['no__urut_ba_tpt__2024_0001_'] ?? null,
'tanggal_ba_tpt' => $this->convertToDate($data['tanggal_ba_tpt']),
'no_urut_ba_tpa' => $data['no__urut_ba_tpa'] ?? null,
'tanggal_ba_tpa' => $this->convertToDate($data['tanggal_ba_tpa']),
'no_urut_skrd' => $data['no__urut_skrd__2024_0001_'] ?? null,
'tanggal_skrd' => $this->convertToDate($data['tanggal_skrd']),
'ptsp' => $data['ptsp'] ?? null,
'selesai_terbit' => $data['selesai_terbit'] ?? null,
'tanggal_pembayaran' => $this->convertToDate($data['tanggal_pembayaran__yyyy_mm_dd_']),
'format_sts' => $data['format_sts'] ?? null,
'tahun_terbit' => (int) $data['tahun_terbit'] ?? null,
'tahun_berjalan' => (int) $data['tahun_berjalan'] ?? null,
'kelurahan' => $data['kelurahan'] ?? null,
'kecamatan' => $data['kecamatan'] ?? null,
'lb' => $this->convertToDecimal($data['lb']) ?? null,
'tb' => $this->convertToDecimal($data['tb']) ?? null,
'jlb' => (int) $data['jlb'] ?? null,
'unit' => (int) $data['unit'] ?? null,
'usulan_retribusi' => (int) $data['usulan_retribusi'] ?? null,
'nilai_retribusi_keseluruhan_simbg' => $this->convertToDecimal($data['nilai_retribusi_keseluruhan__simbg_']) ?? null,
'nilai_retribusi_keseluruhan_pad' => $this->convertToDecimal($data['nilai_retribusi_keseluruhan__pad_']) ?? null,
'denda' => $this->convertToDecimal($data['denda']) ?? null,
'latitude' => $data['latitude'] ?? null,
'longitude' => $data['longitude'] ?? null,
'nik_nib' => $data['nik_nib'] ?? null,
'dok_tanah' => $data['dok__tanah'] ?? null,
'temuan' => $data['temuan'] ?? null,
];
foreach ($sheetData as $data) {
$mapToUpsert[] = [
'no_registrasi' => $this->cleanString($data['no__registrasi'] ?? null),
'jenis_konsultasi' => $this->cleanString($data['jenis_konsultasi'] ?? null),
'fungsi_bg' => $this->cleanString($data['fungsi_bg'] ?? null),
'tgl_permohonan' => $this->convertToDate($this->cleanString($data['tgl_permohonan'] ?? null)),
'status_verifikasi' => $this->cleanString($data['status_verifikasi'] ?? null),
'status_permohonan' => $this->convertToDate($this->cleanString($data['status_permohonan'] ?? null)),
'alamat_pemilik' => $this->cleanString($data['alamat_pemilik'] ?? null),
'no_hp' => $this->cleanString($data['no__hp'] ?? null),
'email' => $this->cleanString($data['e_mail'] ?? null),
'tanggal_catatan' => $this->convertToDate($this->cleanString($data['tanggal_catatan'] ?? null)),
'catatan_kekurangan_dokumen' => $this->cleanString($data['catatan_kekurangan_dokumen'] ?? null),
'gambar' => $this->cleanString($data['gambar'] ?? null),
'krk_kkpr' => $this->cleanString($data['krk_kkpr'] ?? null),
'no_krk' => $this->cleanString($data['no__krk'] ?? null),
'lh' => $this->cleanString($data['lh'] ?? null),
'ska' => $this->cleanString($data['ska'] ?? null),
'keterangan' => $this->cleanString($data['keterangan'] ?? null),
'helpdesk' => $this->cleanString($data['helpdesk'] ?? null),
'pj' => $this->cleanString($data['pj'] ?? null),
'kepemilikan' => $this->cleanString($data['kepemilikan'] ?? null),
'potensi_taru' => $this->cleanString($data['potensi_taru'] ?? null),
'validasi_dinas' => $this->cleanString($data['validasi_dinas'] ?? null),
'kategori_retribusi' => $this->cleanString($data['kategori_retribusi'] ?? null),
'no_urut_ba_tpt' => $this->cleanString($data['no__urut_ba_tpt__2024_0001_'] ?? null),
'tanggal_ba_tpt' => $this->convertToDate($this->cleanString($data['tanggal_ba_tpt'] ?? null)),
'no_urut_ba_tpa' => $this->cleanString($data['no__urut_ba_tpa'] ?? null),
'tanggal_ba_tpa' => $this->convertToDate($this->cleanString($data['tanggal_ba_tpa'] ?? null)),
'no_urut_skrd' => $this->cleanString($data['no__urut_skrd__2024_0001_'] ?? null),
'tanggal_skrd' => $this->convertToDate($this->cleanString($data['tanggal_skrd'] ?? null)),
'ptsp' => $this->cleanString($data['ptsp'] ?? null),
'selesai_terbit' => $this->cleanString($data['selesai_terbit'] ?? null),
'tanggal_pembayaran' => $this->convertToDate($this->cleanString($data['tanggal_pembayaran__yyyy_mm_dd_'] ?? null)),
'format_sts' => $this->cleanString($data['format_sts'] ?? null),
'tahun_terbit' => (int) ($data['tahun_terbit'] ?? null),
'tahun_berjalan' => (int) ($data['tahun_berjalan'] ?? null),
'kelurahan' => $this->cleanString($data['kelurahan'] ?? null),
'kecamatan' => $this->cleanString($data['kecamatan'] ?? null),
'lb' => $this->convertToDecimal($data['lb'] ?? null),
'tb' => $this->convertToDecimal($data['tb'] ?? null),
'jlb' => (int) ($data['jlb'] ?? null),
'unit' => (int) ($data['unit'] ?? null),
'usulan_retribusi' => (int) ($data['usulan_retribusi'] ?? null),
'nilai_retribusi_keseluruhan_simbg' => $this->convertToDecimal($data['nilai_retribusi_keseluruhan__simbg_'] ?? null),
'nilai_retribusi_keseluruhan_pad' => $this->convertToDecimal($data['nilai_retribusi_keseluruhan__pad_'] ?? null),
'denda' => $this->convertToDecimal($data['denda'] ?? null),
'latitude' => $this->cleanString($data['latitude'] ?? null),
'longitude' => $this->cleanString($data['longitude'] ?? null),
'nik_nib' => $this->cleanString($data['nik_nib'] ?? null),
'dok_tanah' => $this->cleanString($data['dok__tanah'] ?? null),
'temuan' => $this->cleanString($data['temuan'] ?? null),
];
}
$batchSize = 1000;
@@ -656,4 +655,9 @@ class ServiceSIMBG
return null;
}
}
private function cleanString($value)
{
return isset($value) ? trim(strip_tags($value)) : null;
}
}

79
app/Services/ServiceTokenSIMBG.php Normal file
View File

@@ -0,0 +1,79 @@
<?php
namespace App\Services;
use App\Models\GlobalSetting;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\RequestException;
use Illuminate\Support\Facades\Log;
class ServiceTokenSIMBG
{
private $client;
private $login_url;
private $email;
private $password;
private $simbg_host;
private $fetch_per_page;
private $refresh_url;
public function __construct()
{
$settings = GlobalSetting::whereIn('key', [
'SIMBG_EMAIL', 'SIMBG_PASSWORD', 'SIMBG_HOST', 'FETCH_PER_PAGE'
])->pluck('value', 'key');
$this->email = trim((string) ($settings['SIMBG_EMAIL'] ?? ""));
$this->password = trim((string) ($settings['SIMBG_PASSWORD'] ?? ""));
$this->simbg_host = trim((string) ($settings['SIMBG_HOST'] ?? ""));
$this->fetch_per_page = trim((string) ($settings['FETCH_PER_PAGE'] ?? ""));
$this->client = new Client();
$this->login_url = $this->simbg_host . "/api/user/v1/auth/login/";
$this->refresh_url = $this->simbg_host. "/api/user/v1/auth/token/refresh/";
}
public function get_token(){
try {
$response = $this->client->request('POST', $this->login_url, [
'headers' => [
'Content-Type' => 'application/json',
'Accept' => 'application/json',
],
'json' => [
'email' => $this->email,
'password' => $this->password
]
]);
$data = json_decode($response->getBody()->getContents(), true);
return $data['token'];
} catch (RequestException $e) {
Log::error("Failed to get token", [
'error' => $e->getMessage(),
'response' => $e->getResponse() ? $e->getResponse()->getBody()->getContents() : null
]);
return null;
}
}
public function refresh_token(string $refresh_token){
try {
$response = $this->client->request('POST', $this->refresh_url, [
'headers' => [
'Content-Type' => 'application/json',
'Accept' => 'application/json',
],
'json' => [
'refresh' => $refresh_token
]
]);
$data = json_decode($response->getBody()->getContents(), true);
return $data;
} catch (\Throwable $th) {
Log::error("Failed to refresh token", [
'error' => $th->getMessage()
]);
return null;
}
}
}