fix query scraping using upsert and connect dashboard

2025-01-30 22:57:15 +07:00
parent 1f6ef5f110
commit d70fefb12d
9 changed files with 1113 additions and 976 deletions
--- a/app/Http/Controllers/Api/DashboardController.php
+++ b/app/Http/Controllers/Api/DashboardController.php
@@ -69,4 +69,29 @@ class DashboardController extends Controller
        ];
        return $this->resSuccess($result);
    }
+
+    public function pbgTaskDocuments(Request $request){
+        $request->validate([
+            'status' => 'required|string'
+        ]);
+        
+        $businessData = DB::table('pbg_task')
+        ->leftJoin('pbg_task_retributions', 'pbg_task.uuid', '=', 'pbg_task_retributions.pbg_task_uid')
+        ->select(
+            DB::raw('COUNT(DISTINCT pbg_task.id) as task_count'),
+            DB::raw('SUM(pbg_task_retributions.nilai_retribusi_bangunan) as total_retribution')
+        )
+        ->where(function ($query) use ($request) {
+            $query->where("pbg_task.status", "=", $request->get('status'));
+        })
+        ->first();
+        $taskCount = $businessData->task_count;
+        $taskTotal = $businessData->total_retribution;
+        $result = [
+            "count" => $taskCount,
+            "series" => [$taskCount],
+            "total" => $taskTotal
+        ];
+        return $this->resSuccess($result);
+    }
 }
--- a/app/ServiceSIMBG.php
+++ b/app/ServiceSIMBG.php
@@ -12,6 +12,7 @@ use Exception;
 use App\Models\PbgTask;
 use App\Traits\GlobalApiResponse;
 use Illuminate\Support\Facades\Log;
+use Carbon\Carbon;

 class ServiceSIMBG
 {
@@ -19,6 +20,7 @@ class ServiceSIMBG
    private $email;
    private $password;
    private $simbg_host;
+    private $fetch_per_page;
    /**
     * Create a new class instance.
     */
@@ -27,6 +29,7 @@ class ServiceSIMBG
        $this->email = trim((string) GlobalSetting::where('key','SIMBG_EMAIL')->first()->value);
        $this->password = trim((string) GlobalSetting::where('key','SIMBG_PASSWORD')->first()->value);
        $this->simbg_host = trim((string)GlobalSetting::where('key','SIMBG_HOST')->first()->value);
+        $this->fetch_per_page = trim((string)GlobalSetting::where('key','FETCH_PER_PAGE')->first()->value);
    }

    public function getToken(){
@@ -92,41 +95,32 @@ class ServiceSIMBG
        // Log success
        Log::info("syncIndexIntegration completed successfully", ['uuid' => $uuid]);
    }
-    

    public function syncTaskList()
    {
        $clientHelper = new ServiceClient($this->simbg_host);
        $resToken = $this->getToken();

-        // create log import datasource 
        $importDatasource = ImportDatasource::create([
            'status' => ImportDatasourceStatus::Processing->value,
        ]);

-        if (!isset($resToken) || empty($resToken->original['data']['token']['access'])) {
-            Log::error("Token not retrieved for syncTaskList");
+        if (empty($resToken->original['data']['token']['access'])) {
            $importDatasource->update([
                'status' => ImportDatasourceStatus::Failed->value,
-                'message' => 'Failed to retrive token'
+                'message' => 'Failed to retrieve token'
            ]);
-            return $this->resError("Failed to retrive token");
+            return $this->resError("Failed to retrieve token");
        }

        $apiToken = $resToken->original['data']['token']['access'];
-        $queryParams = http_build_query([
-            'page' => 1,
-            'size' => 20,
-            'sort' => 'ASC',
-            // 'type' => 'task',
-        ]);
-
-        $url = "/api/pbg/v1/list/?" . $queryParams;
        $headers = ['Authorization' => "Bearer " . $apiToken];

+        $url = "/api/pbg/v1/list/?page=1&size={$this->fetch_per_page}&sort=ASC";
        $initialResponse = $clientHelper->get($url, $headers);
-        if (empty($initialResponse->original['data']['total_page'])) {
-            Log::error("Invalid response: no total_page", ['response' => $initialResponse->original]);
+
+        $totalPage = $initialResponse->original['data']['total_page'] ?? 0;
+        if ($totalPage === 0) {
            $importDatasource->update([
                'status' => ImportDatasourceStatus::Failed->value,
                'message' => 'Invalid response: no total_page'
@@ -134,97 +128,76 @@ class ServiceSIMBG
            return $this->resError("Invalid response from API");
        }

-        $totalPage = $initialResponse->original['data']['total_page'];
-        $savedCount = 0;
-        $failedCount = 0;
+        $savedCount = $failedCount = 0;

        for ($currentPage = 1; $currentPage <= $totalPage; $currentPage++) {
-
+            $pageUrl = "/api/pbg/v1/list/?page={$currentPage}&size={$this->fetch_per_page}&sort=ASC";
            $token = $this->getToken();
-            $simbg_token = $token->original['data']['token']['access'];
-            $headers = ['Authorization' => "Bearer " . $simbg_token];
+            $headers = ['Authorization' => "Bearer " . $token];
+            $response = $clientHelper->get($pageUrl, $headers);
+            $tasks = $response->original['data']['data'] ?? [];

-            $queryParams = http_build_query([
-                'page' => $currentPage,
-                'size' => 20,
-                'sort' => 'ASC',
-                // 'type' => 'task'
-            ]);
-
-            $url = "/api/pbg/v1/list/?" . $queryParams;
-            $response = $clientHelper->get($url, $headers);
-
-            if (empty($response->original['data']['data'])) {
+            if (empty($tasks)) {
                Log::warning("No data found on page", ['page' => $currentPage]);
-                $importDatasource->update([
-                    'status' => ImportDatasourceStatus::Success->value,
-                    'message' => 'Success but no data loaded on page'
-                ]);
+                continue;
            }

-            foreach ($response->original['data']['data'] as $item) {
-                try {
-                    $taskCreatedAt = isset($item['created_at']) 
-                    ? \Carbon\Carbon::parse($item['created_at'])->format('Y-m-d H:i:s') 
-                    : null;
-                    PbgTask::updateOrCreate(
-                        [
-                            'uuid' => $item['uid'],
-                        ],
-                        [
-                            'name' => $item['name'],
-                            'owner_name' => $item['owner_name'],
-                            'application_type' => $item['application_type'],
-                            'application_type_name' => $item['application_type_name'],
-                            'condition' => $item['condition'],
-                            'registration_number' => $item['registration_number'],
-                            'document_number' => $item['document_number'],
-                            'address' => $item['address'],
-                            'status' => $item['status'],
-                            'status_name' => $item['status_name'],
-                            'slf_status' => $item['slf_status'] ?? null,
-                            'slf_status_name' => $item['slf_status_name'] ?? null,
-                            'function_type' => $item['function_type'],
-                            'consultation_type' => $item['consultation_type'],
-                            'due_date' => $item['due_date'],
-                            'land_certificate_phase' => $item['land_certificate_phase'],
-                            'task_created_at' => $taskCreatedAt,
-                        ]
-                    );
+            Log::info("executed page", ['page' => $currentPage, 'total' => $totalPage]);
+
+            $tasksCollective = [];
+            foreach ($tasks as $item) {
+                try {
+                    $tasksCollective[] = [
+                        'uuid' => $item['uid'],
+                        'name' => $item['name'],
+                        'owner_name' => $item['owner_name'],
+                        'application_type' => $item['application_type'],
+                        'application_type_name' => $item['application_type_name'],
+                        'condition' => $item['condition'],
+                        'registration_number' => $item['registration_number'],
+                        'document_number' => $item['document_number'],
+                        'address' => $item['address'],
+                        'status' => $item['status'],
+                        'status_name' => $item['status_name'],
+                        'slf_status' => $item['slf_status'] ?? null,
+                        'slf_status_name' => $item['slf_status_name'] ?? null,
+                        'function_type' => $item['function_type'],
+                        'consultation_type' => $item['consultation_type'],
+                        'due_date' => $item['due_date'],
+                        'land_certificate_phase' => $item['land_certificate_phase'],
+                        'task_created_at' => isset($item['created_at']) ? Carbon::parse($item['created_at'])->format('Y-m-d H:i:s') : null,
+                        'updated_at' => now(),
+                        'created_at' => now(),
+                    ];

-                    // Synchronize additional details
                    $this->syncIndexIntegration($item['uid']);
                    $this->syncTaskDetailSubmit($item['uid']);
-                    Log::info("executed page: ". $currentPage);
                    $savedCount++;
                } catch (Exception $e) {
-                    $importDatasource->update([
-                        'status' => ImportDatasourceStatus::Failed->value,
-                        'message' => 'failed to save',
-                        'response_body' => $item
-                    ]);
+                    $failedCount++;
                    Log::error("Failed to process task", [
                        'error' => $e->getMessage(),
                        'task' => $item,
                    ]);
-                    $failedCount++;
                }
            }
+
+            PbgTask::upsert($tasksCollective, ['uuid'], [
+                'name', 'owner_name', 'application_type', 'application_type_name', 'condition',
+                'registration_number', 'document_number', 'address', 'status', 'status_name',
+                'slf_status', 'slf_status_name', 'function_type', 'consultation_type', 'due_date',
+                'land_certificate_phase', 'task_created_at', 'updated_at'
+            ]);
        }

-        $result = [
-            "savedCount" => $savedCount,
-            "failedCount" => $failedCount,
-        ];
-
-        
        $importDatasource->update([
            'status' => ImportDatasourceStatus::Success->value,
-            'message' => "Successfully success data: " .$savedCount. " failed data : " .$failedCount
+            'message' => "Successfully processed: $savedCount, Failed: $failedCount"
        ]);
-        
-        Log::info("syncTaskList completed", $result);
-        return $this->resSuccess($result);
+
+        Log::info("syncTaskList completed", ['savedCount' => $savedCount, 'failedCount' => $failedCount]);
+
+        return $this->resSuccess(['savedCount' => $savedCount, 'failedCount' => $failedCount]);
    }


@@ -260,11 +233,11 @@ class ServiceSIMBG
        }

        $detailCreatedAt = isset($data['created_at']) 
-        ? \Carbon\Carbon::parse($data['created_at'])->format('Y-m-d H:i:s') 
+        ? Carbon::parse($data['created_at'])->format('Y-m-d H:i:s') 
        : null;

        $detailUpdatedAt = isset($data['updated_at']) 
-        ? \Carbon\Carbon::parse($data['updated_at'])->format('Y-m-d H:i:s') 
+        ? Carbon::parse($data['updated_at'])->format('Y-m-d H:i:s') 
        : null;

        PbgTaskRetributions::updateOrCreate(
@@ -292,24 +265,20 @@ class ServiceSIMBG
        );

        $prasaranaData = $data['prasarana'] ?? [];
-        if (is_array($prasaranaData) && count($prasaranaData) > 0) {
-            foreach ($prasaranaData as $item) {
-                PbgTaskPrasarana::updateOrCreate(
-                    [
-                        'pbg_task_uid' => $uuid,
-                        'prasarana_id' => $item['id'] ?? null,
-                    ],
-                    [
-                        'pbg_task_uid' => $uuid,
-                        'prasarana_type' => $item['prasarana_type'] ?? null,
-                        'building_type' => $item['building_type'] ?? null,
-                        'total' => $item['total'] ?? null,
-                        'quantity' => $item['quantity'] ?? null,
-                        'unit' => $item['unit'] ?? null,
-                        'index_prasarana' => $item['index_prasarana'] ?? null,
-                    ]
-                );
-            }
+        if (!empty($prasaranaData)) {
+            $insertData = array_map(fn($item) => [
+                'pbg_task_uid' => $uuid,
+                'prasarana_id' => $item['id'] ?? null,
+                'prasarana_type' => $item['prasarana_type'] ?? null,
+                'building_type' => $item['building_type'] ?? null,
+                'total' => $item['total'] ?? null,
+                'quantity' => $item['quantity'] ?? null,
+                'unit' => $item['unit'] ?? null,
+                'index_prasarana' => $item['index_prasarana'] ?? null,
+            ], $prasaranaData);
+    
+            // Use bulk insert or upsert for faster database operation
+            PbgTaskPrasarana::upsert($insertData, ['pbg_task_uid', 'prasarana_id']);
        }

        Log::info("syncTaskDetailSubmit completed successfully", ['uuid' => $uuid]);
--- a/database/seeders/GlobalSettingSeeder.php
+++ b/database/seeders/GlobalSettingSeeder.php
@@ -39,7 +39,26 @@ class GlobalSettingSeeder extends Seeder
                "created_at" => Carbon::now()->format("Y-m-d H:i:s"),
                "updated_at" => Carbon::now()->format("Y-m-d H:i:s"),
            ],
+            [
+                "key" => "FETCH_PER_PAGE",
+                "value" => "100",
+                "type" => "integer",
+                "description" => "Total data per page",
+                "created_at" => Carbon::now()->format("Y-m-d H:i:s"),
+                "updated_at" => Carbon::now()->format("Y-m-d H:i:s"),
+            ],
        ];
-        GlobalSetting::insert($globalSettings);
+        foreach($globalSettings as $setting){
+            GlobalSetting::updateOrCreate(
+                ["key" => $setting["key"]],
+                [
+                    "value" =>$setting["value"],
+                    "type" => $setting["type"],
+                    "description" => $setting["description"],
+                    "created_at" => Carbon::now()->format("Y-m-d H:i:s"),
+                    "updated_at" => Carbon::now()->format("Y-m-d H:i:s"),
+                ]
+            );
+        }
    }
 }
--- a/public/images/simbg-dputr.png
+++ b/public/images/simbg-dputr.png
--- a/resources/js/config.js
+++ b/resources/js/config.js
@@ -2,7 +2,7 @@
    var t = sessionStorage.getItem("__DARKONE_CONFIG__"),
        e = document.getElementsByTagName("html")[0],
        o = {
-            theme: "dark",
+            theme: "light",
            topbar: { color: "light" },
            menu: { size: "default", color: "light" },
        };
--- a/resources/js/dashboards/bigdata.js
+++ b/resources/js/dashboards/bigdata.js
--- a/resources/views/auth/signin.blade.php
+++ b/resources/views/auth/signin.blade.php
@@ -14,11 +14,11 @@ class="authentication-bg"
                        <div class="text-center">
                            <div class="mx-auto mb-4 text-center auth-logo">
                                <a href="{{ route('home') }}" class="logo-dark">
-                                    <img src="/images/logo-dark.png" height="32" alt="logo dark">
+                                    <img src="/images/dputr-kab-bandung.png" height="auto" width="100%" alt="logo dark">
                                </a>

                                <a href="{{ route('home') }}" class="logo-light">
-                                    <img src="/images/logo-light.png" height="28" alt="logo light">
+                                    <img src="/images/dputr-kab-bandung.png" height="auto" width="100%" alt="logo light">
                                </a>
                            </div>
                            <h4 class="fw-bold text-dark mb-2">Welcome Back!</h3>
--- a/resources/views/dashboards/bigdata.blade.php
+++ b/resources/views/dashboards/bigdata.blade.php
@@ -12,12 +12,12 @@
                <div class="row">
                    <div class="col-12">
                        <p class="text-muted mb-0 text-truncate">Total Potensi Berkas</p>
-                        <h5 class="text-dark mt-2 mb-0">2432 Pemohon</h5>
-                        <h3 class="text-dark mt-2 mb-0">Rp.24.416.920.070</h3>
+                        <h5 class="text-dark mt-2 mb-0" id="countStatus1"></h5>
+                        <h3 class="text-dark mt-2 mb-0" id="totalStatus1"></h3>
                    </div>
                </div>
            </div>
-            <div id="chart01"></div>
+            <div id="char-pbg-status-1"></div>
        </div>
    </div>

@@ -28,12 +28,12 @@
                <div class="row">
                    <div class="col-12">
                        <p class="text-muted mb-0 text-truncate">Berkas Terverifikasi</p>
-                        <h5 class="text-dark mt-2 mb-0">1572 Berkas</h5>
-                        <h3 class="text-dark mt-2 mb-0">Rp.17.522.994.118</h3>
+                        <h5 class="text-dark mt-2 mb-0" id="countStatus2"></h5>
+                        <h3 class="text-dark mt-2 mb-0" id="totalStatus2"></h3>
                    </div>
                </div>
            </div>
-            <div id="chart02"></div>
+            <div id="chart-pbg-status-2"></div>
        </div>
    </div>

@@ -44,12 +44,12 @@
                <div class="row">
                    <div class="col-12">
                        <p class="text-muted mb-0 text-truncate">Berkas Belum Terverifikasi</p>
-                        <h5 class="text-dark mt-2 mb-0">860 Berkas</h5>
-                        <h3 class="text-dark mt-2 mb-0">Rp.6.893.925.952</h3>
+                        <h5 class="text-dark mt-2 mb-0" id="countStatus3"></h5>
+                        <h3 class="text-dark mt-2 mb-0" id="totalStatus3"></h3>
                    </div>
                </div>
            </div>
-            <div id="chart03"></div>
+            <div id="chart-pbg-status-3"></div>
        </div>
    </div>

@@ -94,12 +94,12 @@
                <div class="row">
                    <div class="col-12">
                        <p class="text-muted mb-0 text-truncate">Berproses Di Dinas Teknis</p>
-                        <h5 class="text-dark mt-2 mb-0">171 Berkas</h5>
-                        <h3 class="text-dark mt-2 mb-0">Rp.1.973.666.055</h3>
+                        <h5 class="text-dark mt-2 mb-0" id="countStatus4"></h5>
+                        <h3 class="text-dark mt-2 mb-0" id="totalStatus4"></h3>
                    </div>
                </div>
            </div>
-            <div id="chart06"></div>
+            <div id="chart-pbg-status-4"></div>
        </div>
    </div>

@@ -110,12 +110,12 @@
                <div class="row">
                    <div class="col-12">
                        <p class="text-muted mb-0 text-truncate">Berproses Di DPMPTSP</p>
-                        <h5 class="text-dark mt-2 mb-0">23 Berkas</h5>
-                        <h3 class="text-dark mt-2 mb-0">Rp.1.086.206.621</h3>
+                        <h5 class="text-dark mt-2 mb-0" id="countStatus5"></h5>
+                        <h3 class="text-dark mt-2 mb-0" id="totalStatus5"></h3>
                    </div>
                </div>
            </div>
-            <div id="chart07"></div>
+            <div id="chart-pbg-status-5"></div>
        </div>
    </div>

@@ -126,12 +126,12 @@
                <div class="row">
                    <div class="col-12">
                        <p class="text-muted mb-0 text-truncate">Realisasi Terbit PBG</p>
-                        <h5 class="text-dark mt-2 mb-0">1378 Berkas</h5>
-                        <h3 class="text-dark mt-2 mb-0">Rp.14.463.121.442</h3>
+                        <h5 class="text-dark mt-2 mb-0" id="countStatus6"></h5>
+                        <h3 class="text-dark mt-2 mb-0" id="totalStatus6"></h3>
                    </div>
                </div>
            </div>
-            <div id="chart08"></div>
+            <div id="chart-pbg-status-6"></div>
        </div>
    </div>
 </div>
@@ -144,12 +144,12 @@
                <div class="row">
                    <div class="col-12">
                        <p class="text-muted mb-0 text-truncate">Progres Manual</p>
-                        <h5 class="text-dark mt-2 mb-0">85 Berkas</h5>
-                        <h3 class="text-dark mt-2 mb-0">Rp.1.479.160.749</h3>
+                        <h5 class="text-dark mt-2 mb-0" id="countStatus7"></h5>
+                        <h3 class="text-dark mt-2 mb-0" id="totalStatus7"></h3>
                    </div>
                </div>
            </div>
-            <div id="chart09"></div>
+            <div id="chart-pbg-status-7"></div>
        </div>
    </div>

@@ -160,12 +160,12 @@
                <div class="row">
                    <div class="col-12">
                        <p class="text-muted mb-0 text-truncate">Perkiraan Potensi PBG Dari Tata Ruang</p>
-                        <h5 class="text-dark mt-2 mb-0">5 Berkas</h5>
-                        <h3 class="text-dark mt-2 mb-0">Rp.1.898.364.080</h3>
+                        <h5 class="text-dark mt-2 mb-0" id="countStatus20"></h5>
+                        <h3 class="text-dark mt-2 mb-0" id="totalStatus20"></h3>
                    </div>
                </div>
            </div>
-            <div id="chart10"></div>
+            <div id="chart-pbg-status-20"></div>
        </div>
    </div>

@@ -176,12 +176,12 @@
                <div class="row">
                    <div class="col-12">
                        <p class="text-muted mb-0 text-truncate">Perkiraan Potensi PBG Dari Tata Ruang</p>
-                        <h5 class="text-dark mt-2 mb-0">18 Berkas</h5>
-                        <h3 class="text-dark mt-2 mb-0">Rp.3.076.196.000</h3>
+                        <h5 class="text-dark mt-2 mb-0" id="countStatus24"></h5>
+                        <h3 class="text-dark mt-2 mb-0" id="totalStatus24"></h3>
                    </div>
                </div>
            </div>
-            <div id="chart11"></div>
+            <div id="chart-pbg-status-24"></div>
        </div>
    </div>

--- a/routes/api.php
+++ b/routes/api.php
@@ -29,11 +29,14 @@ Route::group(['middleware' => 'auth:sanctum'], function (){

    // request assignments
    Route::apiResource('request-assignments',RequestAssignmentController::class);
-});
-Route::controller(DashboardController::class)->group(function(){
-    Route::get('/business-documents','businnessDocument');
-    Route::get('/non-business-documents','nonBusinnessDocument');
-    Route::get('/all-task-documents', 'allTaskDocuments');
+
+    // all dashboards
+    Route::controller(DashboardController::class)->group(function(){
+        Route::get('/business-documents','businnessDocument');
+        Route::get('/non-business-documents','nonBusinnessDocument');
+        Route::get('/all-task-documents', 'allTaskDocuments');
+        Route::get('/pbg-task-documents', 'pbgTaskDocuments');
+    });
 });