arifal/sibedas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix queue execute scraping syncronize simbg add new column to resume

Browse Source
This commit is contained in:
arifal
2025-03-03 22:55:57 +07:00
parent 9ea7e96af1
commit 3bfcaddba4
16 changed files with 503 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
app/Console/Commands/ExecuteScraping.php
View File

@@ -2,6 +2,7 @@
namespace App\Console\Commands;
use App\Jobs\SyncronizeSIMBG;
use App\Services\ServiceSIMBG;
use Illuminate\Console\Command;
use \Illuminate\Support\Facades\Log;
@@ -28,13 +29,12 @@ class ExecuteScraping extends Command
private $service_simbg;
public function __construct(ServiceSIMBG $service_simbg){
$this->service_simbg = $service_simbg;
public function __construct(){
parent::__construct();
}
public function handle()
{
SyncronizeSIMBG::dispatch();
Log::info("running scheduler daily scraping");
$this->service_simbg->syncTaskList();
}
}

5
app/Http/Controllers/Api/LackOfPotentialController.php
View File

@@ -7,6 +7,7 @@ use App\Models\Advertisement;
use App\Models\Customer;
use App\Models\SpatialPlanning;
use Illuminate\Http\Request;
use App\Models\TourismBasedKBLI;
class LackOfPotentialController extends Controller
{
@@ -16,11 +17,13 @@ class LackOfPotentialController extends Controller
$total_reklame = Advertisement::count();
$total_pdam = Customer::count();
$total_tata_ruang = SpatialPlanning::count();
$data_report_tourism = TourismBasedKBLI::all();
return response()->json([
'total_reklame' => $total_reklame,
'total_pdam' => $total_pdam,
'total_tata_ruang' => $total_tata_ruang
'total_tata_ruang' => $total_tata_ruang,
'data_report' => $data_report_tourism,
], 200);
}catch(\Exception $e){
return response()->json([

5
app/Http/Controllers/Api/ScrapingController.php
View File

@@ -4,6 +4,7 @@ namespace App\Http\Controllers\Api;
use App\Enums\ImportDatasourceStatus;
use App\Http\Controllers\Controller;
use App\Jobs\SyncronizeSIMBG;
use App\Models\ImportDatasource;
use App\Traits\GlobalApiResponse;
use Illuminate\Support\Facades\Artisan;
@@ -23,8 +24,8 @@ class ScrapingController extends Controller
}
// run service artisan command
Artisan::call("app:execute-scraping");
return $this->resSuccess("Success execute scraping service please wait");
SyncronizeSIMBG::dispatch();
return $this->resSuccess(["message" => "Success execute scraping service on background, check status for more"]);
}
/**

4
app/Http/Controllers/Settings/SyncronizeController.php
View File

@@ -17,12 +17,12 @@ class SyncronizeController extends Controller
}
public function syncPbgTask(){
$res = $this->service_simbg->syncTaskList();
$res = $this->service_simbg->syncTaskPBG();
return $res;
}
public function syncronizeTask(Request $request){
$res = $this->service_simbg->syncTaskList();
$res = $this->service_simbg->syncTaskPBG();
return redirect()->back()->with('success', 'Processing completed successfully');
}

27
app/Jobs/SyncronizeSIMBG.php Normal file
View File

@@ -0,0 +1,27 @@
<?php
namespace App\Jobs;
use App\Services\GoogleSheetService;
use App\Services\ServiceSIMBG;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
class SyncronizeSIMBG implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public function __construct()
{
}
public function handle(): void
{
$serviceSIMBG = app(ServiceSIMBG::class);
$serviceSIMBG->syncTaskPBG();
}
}

32
app/Models/BigdataResume.php
View File

@@ -23,7 +23,13 @@ class BigdataResume extends Model
'non_business_sum',
'spatial_count',
'spatial_sum',
'year'
'year',
'waiting_click_dpmptsp_count',
'waiting_click_dpmptsp_sum',
'issuance_realization_pbg_count',
'issuance_realization_pbg_sum',
'process_in_technical_office_count',
'process_in_technical_office_sum',
];
public function importDatasource()
@@ -31,7 +37,7 @@ class BigdataResume extends Model
return $this->belongsTo(ImportDatasource::class, 'import_datasource_id');
}
public static function generateResumeData($import_datasource_id, $year){
public static function generateResumeData($import_datasource_id, $year, $data_setting){
$stats = PbgTask::with(['googleSheet', 'pbg_task_retributions'])
->leftJoin('pbg_task_retributions as ptr', 'pbg_task.uuid', '=', 'ptr.pbg_task_uid')
->leftJoin('pbg_task_google_sheet as ptgs', 'pbg_task.registration_number', '=', 'ptgs.no_registrasi')
@@ -82,9 +88,15 @@ class BigdataResume extends Model
$potention_total = $query_potention->total_retribution ?? 0;
$query_spatial_plannings = once(function () use ($year) {
$query = PbgTask::join('spatial_plannings as sp', 'pbg_task.document_number', '=', 'sp.number')
->join('pbg_task_retributions as ptr', 'ptr.pbg_task_uid', '=', 'pbg_task.uuid')
->selectRaw('COUNT(DISTINCT pbg_task.id) as task_count, SUM(ptr.nilai_retribusi_bangunan) as total_retribution');
$query = PbgTask::leftJoin('spatial_plannings as sp', 'pbg_task.document_number', '=', 'sp.number')
->leftJoin('pbg_task_retributions as ptr', 'ptr.pbg_task_uid', '=', 'pbg_task.uuid')
->selectRaw('
CASE
WHEN COUNT(DISTINCT sp.id) > 0 THEN COUNT(DISTINCT sp.id)
ELSE (SELECT COUNT(*) FROM spatial_plannings)
END as task_count,
SUM(CASE WHEN sp.id IS NOT NULL AND ptr.id IS NOT NULL THEN ptr.nilai_retribusi_bangunan ELSE 0 END) as total_retribution
');
if ($year !== 'all') {
$query->whereYear('pbg_task.task_created_at', (int) $year);
@@ -94,7 +106,7 @@ class BigdataResume extends Model
});
$spatial_planning_count = $query_spatial_plannings->task_count ?? 0;
$spatial_planning_total = $query_spatial_plannings->total_retribution ?? 0;
$spatial_planning_total = $query_spatial_plannings->total_retribution;
$potention_count -= $spatial_planning_count;
$potention_total -= $spatial_planning_total;
@@ -113,7 +125,13 @@ class BigdataResume extends Model
'business_sum' => $business_total ?? 0.00,
'non_business_count' => $non_business_count ?? 0,
'non_business_sum' => $non_business_total ?? 0.00,
'year' => $year
'year' => $year,
'waiting_click_dpmptsp_count' => $data_setting['MENUNGGU_KLIK_DPMPTSP_COUNT'] ?? 0,
'waiting_click_dpmptsp_sum' => $data_setting['MENUNGGU_KLIK_DPMPTSP_SUM'] ?? 0.00,
'issuance_realization_pbg_count' => $data_setting['REALISASI_TERBIT_PBG_COUNT'] ?? 0,
'issuance_realization_pbg_sum' => $data_setting['REALISASI_TERBIT_PBG_SUM'] ?? 0.00,
'process_in_technical_office_count' => $data_setting['PROSES_DINAS_TEKNIS_COUNT'] ?? 0,
'process_in_technical_office_sum' => $data_setting['PROSES_DINAS_TEKNIS_SUM'] ??0.00,
]);
}
}

6
app/Providers/AppServiceProvider.php
View File

@@ -9,6 +9,8 @@ use Illuminate\Support\Facades\Blade;
use Illuminate\Support\Facades\View;
use Illuminate\Support\ServiceProvider;
use Carbon\Carbon;
use App\Services\ServiceSIMBG;
use App\Services\GoogleSheetService;
class AppServiceProvider extends ServiceProvider
{
@@ -17,7 +19,9 @@ class AppServiceProvider extends ServiceProvider
*/
public function register(): void
{
//
$this->app->singleton(ServiceSIMBG::class, function ($app) {
return new ServiceSIMBG($app->make(GoogleSheetService::class));
});
}
/**

1
app/Services/GoogleSheetService.php
View File

@@ -13,6 +13,7 @@ class GoogleSheetService
protected $client;
protected $service;
protected $spreadsheetID;
protected $service_sheets;
public function __construct()
{
$this->client = new Google_Client();

228
app/Services/ServiceSIMBG.php
View File

@@ -15,6 +15,9 @@ use App\Traits\GlobalApiResponse;
use Illuminate\Support\Facades\Log;
use Carbon\Carbon;
use App\Services\ServiceClient;
use App\Services\GoogleSheetService;
use App\Models\DataSetting;
use App\Models\PbgTaskGoogleSheet;
class ServiceSIMBG
{
@@ -24,10 +27,11 @@ class ServiceSIMBG
private $simbg_host;
private $fetch_per_page;
private $service_client;
private $googleSheetService;
/**
* Create a new class instance.
*/
public function __construct()
public function __construct(GoogleSheetService $googleSheetService)
{
$settings = GlobalSetting::whereIn('key', [
'SIMBG_EMAIL', 'SIMBG_PASSWORD', 'SIMBG_HOST', 'FETCH_PER_PAGE'
@@ -39,6 +43,7 @@ class ServiceSIMBG
$this->fetch_per_page = trim((string) ($settings['FETCH_PER_PAGE'] ?? ""));
$this->service_client = new ServiceClient($this->simbg_host);
$this->googleSheetService = $googleSheetService;
}
public function getToken(){
@@ -112,12 +117,173 @@ class ServiceSIMBG
}
}
public function syncTaskList()
public function syncTaskPBG()
{
try {
$importDatasource = ImportDatasource::create([
'status' => ImportDatasourceStatus::Processing->value,
]);
// sync google sheet first
$totalRowCount = $this->googleSheetService->getLastRowByColumn("C");
$sheetData = $this->googleSheetService->getSheetDataCollection($totalRowCount);
$sheet_big_data = $this->googleSheetService->get_data_by_sheet();
$data_setting_result = []; // Initialize result storage
$found_section = null; // Track which section is found
foreach ($sheet_big_data as $row) {
// Check for section headers
if (in_array("•PROSES PENERBITAN:", $row)) {
$found_section = "MENUNGGU_KLIK_DPMPTSP";
} elseif (in_array("•BERKAS AKTUAL TERVERIFIKASI DINAS TEKNIS 2024:", $row)) {
$found_section = "REALISASI_TERBIT_PBG";
} elseif (in_array("•TERPROSES DI DPUTR: belum selesai rekomtek'", $row)) {
$found_section = "PROSES_DINAS_TEKNIS";
}
// If a section is found and we reach "Grand Total", save the corresponding values
if ($found_section && isset($row[0]) && trim($row[0]) === "Grand Total") {
if ($found_section === "MENUNGGU_KLIK_DPMPTSP") {
$data_setting_result["MENUNGGU_KLIK_DPMPTSP_COUNT"] = $row[2] ?? null;
$data_setting_result["MENUNGGU_KLIK_DPMPTSP_SUM"] = $row[3] ?? null;
} elseif ($found_section === "REALISASI_TERBIT_PBG") {
$data_setting_result["REALISASI_TERBIT_PBG_COUNT"] = $row[2] ?? null;
$data_setting_result["REALISASI_TERBIT_PBG_SUM"] = $row[4] ?? null;
} elseif ($found_section === "PROSES_DINAS_TEKNIS") {
$data_setting_result["PROSES_DINAS_TEKNIS_COUNT"] = $row[2] ?? null;
$data_setting_result["PROSES_DINAS_TEKNIS_SUM"] = $row[3] ?? null;
}
// Reset section tracking after capturing "Grand Total"
$found_section = null;
}
}
foreach ($data_setting_result as $key => $value) {
DataSetting::updateOrInsert(
["key" => $key], // Find by key
["value" => $value] // Update or insert value
);
}
$mapToUpsert = [];
$count = 0;
foreach($sheetData as $data){
$mapToUpsert[] =
[
'no_registrasi' => $data['no__registrasi'] ?? null,
'jenis_konsultasi' => $data['jenis_konsultasi'] ?? null,
'fungsi_bg' => $data['fungsi_bg'] ?? null,
'tgl_permohonan' => $this->convertToDate($data['tgl_permohonan']),
'status_verifikasi' => $data['status_verifikasi'] ?? null,
'status_permohonan' => $this->convertToDate($data['status_permohonan']),
'alamat_pemilik' => $data['alamat_pemilik'] ?? null,
'no_hp' => $data['no__hp'] ?? null,
'email' => $data['e_mail'] ?? null,
'tanggal_catatan' => $this->convertToDate($data['tanggal_catatan']),
'catatan_kekurangan_dokumen' => $data['catatan_kekurangan_dokumen'] ?? null,
'gambar' => $data['gambar'] ?? null,
'krk_kkpr' => $data['krk_kkpr'] ?? null,
'no_krk' => $data['no__krk'] ?? null,
'lh' => $data['lh'] ?? null,
'ska' => $data['ska'] ?? null,
'keterangan' => $data['keterangan'] ?? null,
'helpdesk' => $data['helpdesk'] ?? null,
'pj' => $data['pj'] ?? null,
'kepemilikan' => $data['kepemilikan'] ?? null,
'potensi_taru' => $data['potensi_taru'] ?? null,
'validasi_dinas' => $data['validasi_dinas'] ?? null,
'kategori_retribusi' => $data['kategori_retribusi'] ?? null,
'no_urut_ba_tpt' => $data['no__urut_ba_tpt__2024_0001_'] ?? null,
'tanggal_ba_tpt' => $this->convertToDate($data['tanggal_ba_tpt']),
'no_urut_ba_tpa' => $data['no__urut_ba_tpa'] ?? null,
'tanggal_ba_tpa' => $this->convertToDate($data['tanggal_ba_tpa']),
'no_urut_skrd' => $data['no__urut_skrd__2024_0001_'] ?? null,
'tanggal_skrd' => $this->convertToDate($data['tanggal_skrd']),
'ptsp' => $data['ptsp'] ?? null,
'selesai_terbit' => $data['selesai_terbit'] ?? null,
'tanggal_pembayaran' => $this->convertToDate($data['tanggal_pembayaran__yyyy_mm_dd_']),
'format_sts' => $data['format_sts'] ?? null,
'tahun_terbit' => (int) $data['tahun_terbit'] ?? null,
'tahun_berjalan' => (int) $data['tahun_berjalan'] ?? null,
'kelurahan' => $data['kelurahan'] ?? null,
'kecamatan' => $data['kecamatan'] ?? null,
'lb' => $this->convertToDecimal($data['lb']) ?? null,
'tb' => $this->convertToDecimal($data['tb']) ?? null,
'jlb' => (int) $data['jlb'] ?? null,
'unit' => (int) $data['unit'] ?? null,
'usulan_retribusi' => (int) $data['usulan_retribusi'] ?? null,
'nilai_retribusi_keseluruhan_simbg' => $this->convertToDecimal($data['nilai_retribusi_keseluruhan__simbg_']) ?? null,
'nilai_retribusi_keseluruhan_pad' => $this->convertToDecimal($data['nilai_retribusi_keseluruhan__pad_']) ?? null,
'denda' => $this->convertToDecimal($data['denda']) ?? null,
'latitude' => $data['latitude'] ?? null,
'longitude' => $data['longitude'] ?? null,
'nik_nib' => $data['nik_nib'] ?? null,
'dok_tanah' => $data['dok__tanah'] ?? null,
'temuan' => $data['temuan'] ?? null,
];
}
$batchSize = 1000;
$chunks = array_chunk($mapToUpsert, $batchSize);
foreach($chunks as $chunk){
PbgTaskGoogleSheet::upsert($chunk, ["no_registrasi"],[
'jenis_konsultasi',
'nama_pemilik',
'lokasi_bg',
'fungsi_bg',
'nama_bangunan',
'tgl_permohonan',
'status_verifikasi',
'status_permohonan',
'alamat_pemilik',
'no_hp',
'email',
'tanggal_catatan',
'catatan_kekurangan_dokumen',
'gambar',
'krk_kkpr',
'no_krk',
'lh',
'ska',
'keterangan',
'helpdesk',
'pj',
'kepemilikan',
'potensi_taru',
'validasi_dinas',
'kategori_retribusi',
'no_urut_ba_tpt',
'tanggal_ba_tpt',
'no_urut_ba_tpa',
'tanggal_ba_tpa',
'no_urut_skrd',
'tanggal_skrd',
'ptsp',
'selesai_terbit',
'tanggal_pembayaran',
'format_sts',
'tahun_terbit',
'tahun_berjalan',
'kelurahan',
'kecamatan',
'lb',
'tb',
'jlb',
'unit',
'usulan_retribusi',
'nilai_retribusi_keseluruhan_simbg',
'nilai_retribusi_keseluruhan_pad',
'denda',
'latitude',
'longitude',
'nik_nib',
'dok_tanah',
'temuan',
]);
}
$initResToken = $this->getToken();
if (empty($initResToken->original['data']['token']['access'])) {
@@ -216,8 +382,8 @@ class ServiceSIMBG
}
}
BigdataResume::generateResumeData($importDatasource->id, "all");
BigdataResume::generateResumeData($importDatasource->id, now()->year);
BigdataResume::generateResumeData($importDatasource->id, "all", $data_setting_result);
BigdataResume::generateResumeData($importDatasource->id, now()->year, $data_setting_result);
// Final update after processing all pages
$importDatasource->update([
@@ -323,5 +489,59 @@ class ServiceSIMBG
throw $e;
}
}
protected function convertToDecimal(?string $value): ?float
{
if (empty($value)) {
return null; // Return null if the input is empty
}
// Remove all non-numeric characters except comma and dot
$value = preg_replace('/[^0-9,\.]/', '', $value);
// If the number contains both dot (.) and comma (,)
if (strpos($value, '.') !== false && strpos($value, ',') !== false) {
$value = str_replace('.', '', $value); // Remove thousands separator
$value = str_replace(',', '.', $value); // Convert decimal separator to dot
}
// If only a dot is present (assumed as thousands separator)
elseif (strpos($value, '.') !== false) {
$value = str_replace('.', '', $value); // Remove all dots (treat as thousands separators)
}
// If only a comma is present (assumed as decimal separator)
elseif (strpos($value, ',') !== false) {
$value = str_replace(',', '.', $value); // Convert comma to dot (decimal separator)
}
// Ensure the value is numeric before returning
return is_numeric($value) ? (float) number_format((float) $value, 2, '.', '') : null;
}
protected function convertToInteger($value) {
// Check if the value is an empty string, and return null if true
if (trim($value) === "") {
return null;
}
// Otherwise, cast to integer
return (int) $value;
}
protected function convertToDate($dateString)
{
try {
// Check if the string is empty
if (empty($dateString)) {
return null;
}
// Try to parse the date string
$date = Carbon::parse($dateString);
// Return the Carbon instance
return $date->format('Y-m-d');
} catch (Exception $e) {
// Return null if an error occurs during parsing
return null;
}
}
}