add table scraping log and datatable server side

app/Enums/ImportDatasourceStatus.php

@@ -0,0 +1,10 @@
+<?php
+
+namespace App\Enums;
+
+enum ImportDatasourceStatus: string
+{
+    case Processing = 'processing';
+    case Success = 'success';
+    case Failed = 'failed';
+}

app/Http/Controllers/Api/ImportDatasourceController.php

@@ -0,0 +1,58 @@
+<?php
+
+namespace App\Http\Controllers\Api;
+
+use App\Http\Controllers\Controller;
+use App\Http\Resources\ImportDatasourceResource;
+use App\Models\ImportDatasource;
+use Illuminate\Http\Request;
+
+class ImportDatasourceController extends Controller
+{
+    /**
+     * Display a listing of the resource.
+     */
+    public function index(Request $request)
+    {
+        $query = ImportDatasource::query();
+
+        if($request->has("search") && !empty($request->get("search"))){
+            $search = $request->get("search");
+            $query->where('status', 'like', "%".$search."%");
+        }
+        return ImportDatasourceResource::collection($query->paginate()->withQueryString());
+    }
+
+    /**
+     * Store a newly created resource in storage.
+     */
+    public function store(Request $request)
+    {
+        //
+    }
+
+    /**
+     * Display the specified resource.
+     */
+    public function show(string $id)
+    {
+        //
+    }
+
+    /**
+     * Update the specified resource in storage.
+     */
+    public function update(Request $request, string $id)
+    {
+        //
+    }
+
+    /**
+     * Remove the specified resource from storage.
+     */
+    public function destroy(string $id)
+    {
+        //
+    }
+}
+

app/Http/Controllers/Api/UsersController.php

@@ -21,4 +21,7 @@ class UsersController extends Controller
 
         return response(['user' => $user, 'token' => $token], 200);
     }
+    public function index(){
+        return response()->json(User::all());
+    }
 }

app/Http/Resources/ImportDatasourceResource.php

@@ -0,0 +1,26 @@
+<?php
+
+namespace App\Http\Resources;
+
+use Illuminate\Http\Request;
+use Illuminate\Http\Resources\Json\JsonResource;
+
+class ImportDatasourceResource extends JsonResource
+{
+    /**
+     * Transform the resource into an array.
+     *
+     * @return array<string, mixed>
+     */
+    public function toArray(Request $request): array
+    {
+        return [
+            "id"=> $this->id,
+            "message" => $this->message,
+            "response_body" => $this->response_body,
+            "status" => $this->status,
+            "created_at" => $this->created_at->toDateTimeString(),
+            "updated_at" => $this->updated_at->toDateTimeString(),
+        ];
+    }
+}

app/Models/ImportDatasource.php

@@ -0,0 +1,16 @@
+<?php
+
+namespace App\Models;
+
+use Illuminate\Database\Eloquent\Model;
+
+class ImportDatasource extends Model
+{
+    protected $table = 'import_datasources';
+    protected $fillable = [
+        'id',
+        'message',
+        'response_body',
+        'status'
+    ];
+}

app/ServiceSIMBG.php

@@ -2,7 +2,9 @@
 
 namespace App;
 
+use App\Enums\ImportDatasourceStatus;
 use App\Helpers\ApiResponse;
+use App\Models\ImportDatasource;
 use App\Models\PbgTaskIndexIntegrations;
 use App\Models\PbgTaskPrasarana;
 use App\Models\PbgTaskRetributions;
@@ -31,6 +33,7 @@ class ServiceSIMBG
         
         $res = $clientHelper->post($url, $body);
         if(!$res->original['success']){
+            \Log::error("Token not retrieved ", ['response' => $res]);
             return null;
         }
         return $res;
@@ -90,8 +93,17 @@ class ServiceSIMBG
         $clientHelper = new ServiceClient($_ENV['SIMBG_HOST']);
         $resToken = $this->getToken();
 
+        // create log import datasource 
+        $importDatasource = ImportDatasource::create([
+            'status' => ImportDatasourceStatus::Processing->value,
+        ]);
+
         if (!isset($resToken) || empty($resToken->original['data']['token']['access'])) {
             \Log::error("Token not retrieved for syncTaskList");
+            $importDatasource->update([
+                'status' => ImportDatasourceStatus::Failed->value,
+                'message' => 'Failed to retrive token'
+            ]);
             return ApiResponse::errorResponse("Failed to retrieve token", 401);
         }
 
@@ -109,6 +121,10 @@ class ServiceSIMBG
         $initialResponse = $clientHelper->get($url, $headers);
         if (empty($initialResponse->original['data']['total_page'])) {
             \Log::error("Invalid response: no total_page", ['response' => $initialResponse->original]);
+            $importDatasource->update([
+                'status' => ImportDatasourceStatus::Failed->value,
+                'message' => 'Invalid response: no total_page'
+            ]);
             return ApiResponse::errorResponse("Invalid response from API", 400);
         }
 
@@ -129,7 +145,10 @@ class ServiceSIMBG
 
             if (empty($response->original['data']['data'])) {
                 \Log::warning("No data found on page", ['page' => $currentPage]);
-                continue;
+                $importDatasource->update([
+                    'status' => ImportDatasourceStatus::Success->value,
+                    'message' => 'Success but no data loaded on page'
+                ]);
             }
 
             foreach ($response->original['data']['data'] as $item) {
@@ -171,6 +190,11 @@ class ServiceSIMBG
                         'error' => $e->getMessage(),
                         'task' => $item,
                     ]);
+                    $importDatasource->update([
+                        'status' => ImportDatasourceStatus::Failed->value,
+                        'message' => $e->getMessage(),
+                        'response_body' => $item
+                    ]);
                     $failedCount++;
                 }
             }
@@ -181,8 +205,13 @@ class ServiceSIMBG
             "failedCount" => $failedCount,
         ];
 
-        \Log::info("syncTaskList completed", $result);
         
+        $importDatasource->update([
+            'status' => ImportDatasourceStatus::Success->value,
+            'message' => "Successfully success data: " .$savedCount. " failed data : " .$failedCount
+        ]);
+        
+        \Log::info("syncTaskList completed", $result);
         return ApiResponse::successResponse(json_encode($result), "Successfully saved");
     }
 

database/migrations/2025_01_24_063426_create_import_datasources_table.php

@@ -0,0 +1,30 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        Schema::create('import_datasources', function (Blueprint $table) {
+            $table->id();
+            $table->string('message')->nullable();
+            $table->text('response_body')->nullable();
+            $table->string('status')->default('processing');
+            $table->timestamps();
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::dropIfExists('import_datasources');
+    }
+};

resources/js/app.js

@@ -251,7 +251,7 @@ class ThemeLayout {
         e &&
             e.addEventListener("click", function (e) {
                 "light" === n.config.theme
-                    ? n.changeThemeMode("dark")
+                    ? n.changeThemeMode("light")
                     : n.changeThemeMode("light");
             }),
             (e = document.querySelector("#reset-layout")) &&

resources/js/dashboards/bigdata.js

@@ -7,7 +7,7 @@ import GlobalConfig, {addThousandSeparators} from '../global-config.js';
 
 class BigData {
   init(){
-    this.initChartKekuranganPotensi();
+    this.initAllChart();
     this.initChartTargetPAD();
     this.initChartUsaha();
     this.initChartNonUsaha();
@@ -224,7 +224,7 @@ class BigData {
     });
   }
 
-  initChartKekuranganPotensi(){
+  initAllChart(){
     var options1 = {
       chart: {
         type: 'area',

resources/js/settings/syncronize/syncronize.js

@@ -1,7 +1,36 @@
+import { Grid } from "gridjs/dist/gridjs.umd.js";
+import gridjs from 'gridjs/dist/gridjs.umd.js'
+import 'gridjs/dist/gridjs.umd.js'
+import GlobalConfig from "../../global-config.js";
+
 class SyncronizeTask {
   init(){
+    this.initTableImportDatasources();
     this.onSyncSubmit();
   }
+  initTableImportDatasources(){
+    new Grid({
+      columns: [
+          "ID", "Message", "Response", "Status", "Created",
+      ],
+      pagination: {
+          limit: 15,
+          server: {
+               url: (prev, page, limit) => `${prev}?page=${page}`
+          }
+      },
+      sort: true,
+      search: {
+          server: {
+               url: (prev, page, keyword) => `${prev}?page=${page}&search=${keyword}`
+          }
+      },
+      server: {
+          url: `${GlobalConfig.apiHost}/api/import-datasource`,
+          then: data => data.data.map((item) => [item.id, item.message, item.response_body, item.status, item.created_at])
+      }
+    }).render(document.getElementById("table-import-datasources"));
+  }
   onSyncSubmit(){
     const form = document.getElementById("sync-form");
     if(form){

resources/js/tables/common-table.js

@@ -76,7 +76,7 @@ class CommonTable {
   }
 
   CommonTableInitWithFetchApi(){
-    fetch(`${GlobalConfig.apiHost}/users`)
+    fetch(`${GlobalConfig.apiHost}/api/users`)
     .then((response) => response.json())
     .then((data) => {
       console.log("check log response");
@@ -116,7 +116,7 @@ class CommonTable {
         },
         sort: true,
         search: true,
-        data: data.data
+        data: data
       }).render(document.getElementById("common-table"));
     })
     .catch((error) => console.error("Error fetching data: " + error));

resources/views/layouts/partials/topbar.blade.php

@@ -26,8 +26,8 @@
                          <button type="button" class="topbar-button" id="light-dark-mode">
                               <iconify-icon icon="solar:moon-outline"
                                    class="fs-22 align-middle light-mode"></iconify-icon>
-                              <iconify-icon icon="solar:sun-2-outline"
-                                   class="fs-22 align-middle dark-mode"></iconify-icon>
+                              <!-- <iconify-icon icon="solar:sun-2-outline"
+                                   class="fs-22 align-middle dark-mode"></iconify-icon> -->
                          </button>
                     </div>
 

resources/views/settings/syncronize/index.blade.php

@@ -1,5 +1,9 @@
 @extends('layouts.vertical', ['subtitle' => 'Syncronize'])
 
+@section('css')
+@vite(['node_modules/gridjs/dist/theme/mermaid.min.css'])
+@endsection
+
 @section('content')
 
 @include('layouts.partials/page-title', ['title' => 'Settings', 'subtitle' => 'Syncronize'])
@@ -11,6 +15,9 @@
             <button type="submit" class="btn btn-success width-lg" id="btn-sync-submit">Sync SIMBG</button>
         </form>
     </div>
+    <div>
+        <div id="table-import-datasources"></div>
+    </div>
 </div>
 
 @endsection

routes/api.php

@@ -1,6 +1,7 @@
 <?php
 
 use App\Http\Controllers\Api\DashboardController;
+use App\Http\Controllers\Api\ImportDatasourceController;
 use App\Http\Controllers\Api\UsersController;
 use App\Http\Controllers\Settings\SyncronizeController;
 use App\Models\PbgTask;
@@ -21,9 +22,13 @@ Route::controller(DashboardController::class)->group(function(){
     Route::get('/all-task-documents', 'allTaskDocuments');
 });
 
+Route::get('/users', [UsersController::class, 'index'])->name('users');
 Route::post('/login', [UsersController::class, 'login'])->name('api.user.login');
 
 Route::get('/sync-task', [SyncronizeController::class, 'syncPbgTask'])->name('api.task');
 Route::get('/get-user-token', [SyncronizeController::class, 'getUserToken'])->name('api.task.token');
 Route::get('/get-index-integration-retribution/{uuid}', [SyncronizeController::class, 'syncIndexIntegration'])->name('api.task.inntegration');
 Route::get('/sync-task-submit/{uuid}', [SyncronizeController::class, 'syncTaskDetailSubmit'])->name('api.task.submit');
+
+// import datasource
+Route::apiResource('import-datasource',ImportDatasourceController::class);