fix syncronize using worker and add duration syncronize

README.md

@@ -17,10 +17,10 @@ sudo nano /etc/supervisor/conf.d/laravel-worker.conf
 
 [program:laravel-worker]
 process_name=%(program_name)s_%(process_num)02d
-command=php /home/arifal/development/sibedas-pbg-web/artisan queue:work --queue=default --timeout=40000 --tries=1 --sleep=3
+command=php /home/arifal/development/sibedas-pbg-web/artisan queue:work --queue=default --timeout=40000 --tries=1
 autostart=true
 autorestart=true
-numprocs=4
+numprocs=1
 redirect_stderr=true
 stdout_logfile=/home/arifal/development/sibedas-pbg-web/storage/logs/worker.log
 stopasgroup=true

app/Console/Commands/ScrapingData.php

@@ -2,10 +2,12 @@
 
 namespace App\Console\Commands;
 
+use App\Jobs\ScrapingDataJob;
 use App\Models\ImportDatasource;
 use App\Services\ServiceGoogleSheet;
 use App\Services\ServicePbgTask;
 use App\Services\ServiceTabPbgTask;
+use App\Services\ServiceTokenSIMBG;
 use GuzzleHttp\Client; // Import Guzzle Client
 use Illuminate\Console\Command;
 use Illuminate\Support\Facades\Log;
@@ -26,64 +28,68 @@ class ScrapingData extends Command
      * @var string
      */
     protected $description = 'Command description';
-
-    private $client;
-    private $service_pbg_task;
-    private $service_tab_pbg_task;
-
     /**
      * Inject dependencies.
      */
-    public function __construct(Client $client, ServicePbgTask $service_pbg_task, ServiceTabPbgTask $serviceTabPbgTask)
+    public function __construct(
-    {
+    ) {
         parent::__construct();
-        $this->client = $client;
+    }
-        $this->service_pbg_task = $service_pbg_task;
+
-        $this->service_tab_pbg_task = $serviceTabPbgTask;
+    public function handle()
+    {
+        dispatch(new ScrapingDataJob());
+
+        $this->info("Scraping job dispatched successfully");
     }
 
     /**
      * Execute the console command.
      */
-    public function handle()
+    // public function handle()
-    {
+    // {
 
-        try {
+    //     try {
-            // Create a record with "processing" status
+    //         // Create a record with "processing" status
-            $import_datasource = ImportDatasource::create([
+    //         $import_datasource = ImportDatasource::create([
-                'message' => 'Initiating scraping...',
+    //             'message' => 'Initiating scraping...',
-                'response_body' => null,
+    //             'response_body' => null,
-                'status' => 'processing'
+    //             'status' => 'processing',
-            ]);
+    //             'start_time' => now()
+    //         ]);
 
-            // Run the service
+    //         // Run the service
-            $service_google_sheet = new ServiceGoogleSheet();
+    //         $service_google_sheet = new ServiceGoogleSheet();
-            $service_google_sheet->run_service();
+    //         $service_google_sheet->run_service();
 
-            // Run the ServicePbgTask with injected Guzzle Client
+    //         // Run the ServicePbgTask with injected Guzzle Client
-            $this->service_pbg_task->run_service();
+    //         $this->service_pbg_task->run_service();
 
-            // run the service pbg task assignments
+    //         // run the service pbg task assignments
-            $this->service_tab_pbg_task->run_service();
+    //         $this->service_tab_pbg_task->run_service();
 
-            // Update the record status to "success" after completion
+    //         // Update the record status to "success" after completion
-            $import_datasource->update([
+    //         $import_datasource->update([
-                'status' => 'success',
+    //             'status' => 'success',
-                'message' => 'Scraping completed successfully.'
+    //             'message' => 'Scraping completed successfully.',
-            ]);
+    //             'finish_time' => now()
+    //         ]);
 
-        } catch (\Exception $e) {
+    //     } catch (\Exception $e) {
+
+    //         // Log the error for debugging
+    //         Log::error('Scraping failed: ' . $e->getMessage(), ['trace' => $e->getTraceAsString()]);
+
+    //         // Handle errors by updating the status to "failed"
+    //         if (isset($import_datasource)) {
+    //             $import_datasource->update([
+    //                 'status' => 'failed',
+    //                 'response_body' => 'Error: ' . $e->getMessage(),
+    //                 'finish_time' => now()
+    //             ]);
+    //         }
+    //     }
+    // }
 
-            // Log the error for debugging
-            Log::error('Scraping failed: ' . $e->getMessage(), ['trace' => $e->getTraceAsString()]);
 
-            // Handle errors by updating the status to "failed"
-            if (isset($import_datasource)) {
-                $import_datasource->update([
-                    'status' => 'failed',
-                    'response_body' => 'Error: ' . $e->getMessage()
-                ]);
-            }
-        }
-    }
 }

app/Http/Controllers/Api/MenusController.php

@@ -22,7 +22,8 @@ class MenusController extends Controller
             $query = $query->where("name", "like", "%".$request->get("search")."%");
         }
 
-        return response()->json($query->paginate(config('app.paginate_per_page', 50)));
+        // return response()->json($query->paginate(config('app.paginate_per_page', 50)));
+        return MenuResource::collection($query->paginate(config('app.paginate_per_page',50)));
     }
 
     /**

app/Http/Controllers/Api/ScrapingController.php

@@ -4,9 +4,15 @@ namespace App\Http\Controllers\Api;
 
 use App\Enums\ImportDatasourceStatus;
 use App\Http\Controllers\Controller;
+use App\Jobs\ScrapingDataJob;
 use App\Jobs\SyncronizeSIMBG;
 use App\Models\ImportDatasource;
 use App\Traits\GlobalApiResponse;
+use App\Services\ServiceTokenSIMBG;
+use GuzzleHttp\Client;
+use App\Services\ServiceGoogleSheet;
+use App\Services\ServicePbgTask;
+use App\Services\ServiceTabPbgTask;
 use Illuminate\Support\Facades\Artisan;
 use Illuminate\Http\Request;
 
@@ -23,8 +29,11 @@ class ScrapingController extends Controller
             return $this->resError("Failed to execute while processing another scraping");
         }
 
+        // use ole schema synchronization
         // dispatch(new SyncronizeSIMBG());
-        Artisan::call("app:scraping-data");
+
+        // use new schema synchronization
+        dispatch(new ScrapingDataJob());
         return $this->resSuccess(["message" => "Success execute scraping service on background, check status for more"]);
     }
 

app/Http/Resources/ImportDatasourceResource.php

@@ -2,6 +2,7 @@
 
 namespace App\Http\Resources;
 
+use Carbon\Carbon;
 use Illuminate\Http\Request;
 use Illuminate\Http\Resources\Json\JsonResource;
 
@@ -14,11 +15,18 @@ class ImportDatasourceResource extends JsonResource
      */
     public function toArray(Request $request): array
     {
+        $startTime = $this->start_time ? Carbon::parse($this->start_time) : null;
+        $finishTime = $this->finish_time ? Carbon::parse($this->finish_time) : null;
         return [
             "id"=> $this->id,
             "message" => $this->message,
             "response_body" => $this->response_body,
             "status" => $this->status,
+            "start_time" => $startTime ? $startTime->toDateTimeString() : null,
+            "duration" => ($startTime && $finishTime) 
+                ? $finishTime->diff($startTime)->format('%H:%I:%S') 
+                : null,
+            "finish_time" => $finishTime ? $finishTime->toDateTimeString() : null,
             "created_at" => $this->created_at->toDateTimeString(),
             "updated_at" => $this->updated_at->toDateTimeString(),
         ];

app/Http/Resources/MenuResource.php

@@ -14,6 +14,16 @@ class MenuResource extends JsonResource
      */
     public function toArray(Request $request): array
     {
-        return parent::toArray($request);
+        // return parent::toArray($request);
+        return [
+            'id' => $this->id,
+            'name' => $this->name,
+            'icon' => $this->icon,
+            'url' => $this->url,
+            'sort_order' => $this->sort_order,
+            'parent' => $this->parent ? new MenuResource($this->parent) : null,
+            'created_at' => $this->created_at,
+            'updated_at' => $this->updated_at
+        ];
     }
 }

app/Jobs/ScrapingDataJob.php

@@ -0,0 +1,77 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Models\ImportDatasource;
+use App\Services\ServiceGoogleSheet;
+use App\Services\ServicePbgTask;
+use App\Services\ServiceTabPbgTask;
+use App\Services\ServiceTokenSIMBG;
+use GuzzleHttp\Client;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Log;
+
+class ScrapingDataJob implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    /**
+     * Inject dependencies instead of creating them inside.
+     */
+    public function __construct(
+    ) {
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle()
+    {
+        try {
+
+            $client = app(Client::class);
+            $service_pbg_task = app(ServicePbgTask::class);
+            $service_tab_pbg_task = app(ServiceTabPbgTask::class);
+            $service_google_sheet = app(ServiceGoogleSheet::class);
+            $service_token = app(ServiceTokenSIMBG::class);
+            // Create a record with "processing" status
+            $import_datasource = ImportDatasource::create([
+                'message' => 'Initiating scraping...',
+                'response_body' => null,
+                'status' => 'processing',
+                'start_time' => now()
+            ]);
+
+            // Run the scraping services
+            $service_google_sheet->run_service();
+            $service_pbg_task->run_service();
+            $service_tab_pbg_task->run_service();
+
+            // Update status to success
+            $import_datasource->update([
+                'status' => 'success',
+                'message' => 'Scraping completed successfully.',
+                'finish_time' => now()
+            ]);
+
+        } catch (\Exception $e) {
+            Log::error('Scraping failed: ' . $e->getMessage(), ['trace' => $e->getTraceAsString()]);
+
+            // Update status to failed
+            if (isset($import_datasource)) {
+                $import_datasource->update([
+                    'status' => 'failed',
+                    'response_body' => 'Error: ' . $e->getMessage(),
+                    'finish_time' => now()
+                ]);
+            }
+
+            // Mark the job as failed
+            $this->fail($e);
+        }
+    }
+}

app/Models/ImportDatasource.php

@@ -13,6 +13,8 @@ class ImportDatasource extends Model
         'id',
         'message',
         'response_body',
-        'status'
+        'status',
+        'start_time',
+        'finish_time'
     ];
 }

app/Models/Menu.php

@@ -22,4 +22,7 @@ class Menu extends Model
     public function children(){
         return $this->hasMany(Menu::class,'parent_id');
     }
+    public function parent(){
+        return $this->belongsTo(Menu::class,'parent_id');
+    }
 }

app/Providers/AppServiceProvider.php

@@ -3,8 +3,13 @@
 namespace App\Providers;
 
 use App\Models\Menu;
+use App\Services\ServiceGoogleSheet;
+use App\Services\ServicePbgTask;
+use App\Services\ServiceTabPbgTask;
+use App\Services\ServiceTokenSIMBG;
 use App\View\Components\Circle;
 use Auth;
+use GuzzleHttp\Client;
 use Illuminate\Support\Facades\Blade;
 use Illuminate\Support\Facades\View;
 use Illuminate\Support\ServiceProvider;
@@ -19,11 +24,24 @@ class AppServiceProvider extends ServiceProvider
      */
     public function register(): void
     {
-        $this->app->singleton(GoogleSheetService::class, function () {
+        $this->app->bind(Client::class, function () {
-            return new GoogleSheetService();
+            return new Client();
         });
-        $this->app->singleton(ServiceSIMBG::class, function ($app) {
+
-            return new ServiceSIMBG($app->make(GoogleSheetService::class));
+        $this->app->bind(ServiceTokenSIMBG::class, function ($app) {
+            return new ServiceTokenSIMBG();
+        });
+
+        $this->app->bind(ServicePbgTask::class, function ($app) {
+            return new ServicePbgTask($app->make(Client::class), $app->make(ServiceTokenSIMBG::class));
+        });
+
+        $this->app->bind(ServiceTabPbgTask::class, function ($app) {
+            return new ServiceTabPbgTask($app->make(Client::class), $app->make(ServiceTokenSIMBG::class));
+        });
+
+        $this->app->bind(ServiceGoogleSheet::class, function ($app) {
+            return new ServiceGoogleSheet();
         });
     }
 

app/Services/ServiceTabPbgTask.php

@@ -115,10 +115,15 @@ class ServiceTabPbgTask
             } catch (\GuzzleHttp\Exception\ClientException $e) {
                 if ($e->getCode() === 401 && !$retriedAfter401) {
                     Log::warning("401 Unauthorized - Refreshing token and retrying...");
-                    $this->refreshToken();
+                    try{
-                    $options['headers']['Authorization'] = "Bearer {$this->user_token}";
+                        $this->refreshToken();
-                    $retriedAfter401 = true;
+                        $options['headers']['Authorization'] = "Bearer {$this->user_token}";
-                    continue; // Retry with new token
+                        $retriedAfter401 = true;
+                        continue;
+                    }catch(\Exception $refreshError){
+                        Log::error("Token refresh and login failed: " . $refreshError->getMessage());
+                        return false;
+                    }
                 }
 
                 throw $e;
@@ -221,10 +226,15 @@ class ServiceTabPbgTask
             } catch (\GuzzleHttp\Exception\ClientException $e) {
                 if ($e->getCode() === 401 && !$retriedAfter401) {
                     Log::warning("401 Unauthorized - Refreshing token and retrying...");
-                    $this->refreshToken();
+                    try{
-                    $options['headers']['Authorization'] = "Bearer {$this->user_token}";
+                        $this->refreshToken();
-                    $retriedAfter401 = true;
+                        $options['headers']['Authorization'] = "Bearer {$this->user_token}";
-                    continue;
+                        $retriedAfter401 = true;
+                        continue;
+                    }catch(\Exception $refreshError){
+                        Log::error("Token refresh and login failed: " . $refreshError->getMessage());
+                        return false;
+                    }
                 }
 
                 return false;
@@ -295,10 +305,15 @@ class ServiceTabPbgTask
             } catch (\GuzzleHttp\Exception\ClientException $e) {
                 if ($e->getCode() === 401 && !$retriedAfter401) {
                     Log::warning("401 Unauthorized - Refreshing token and retrying...");
-                    $this->refreshToken();
+                    try{
-                    $options['headers']['Authorization'] = "Bearer {$this->user_token}";
+                        $this->refreshToken();
-                    $retriedAfter401 = true;
+                        $options['headers']['Authorization'] = "Bearer {$this->user_token}";
-                    continue;
+                        $retriedAfter401 = true;
+                        continue;
+                    }catch(\Exception $refreshError){
+                        Log::error("Token refresh and login failed: " . $refreshError->getMessage());
+                        return false;
+                    }
                 }
 
                 return false;
@@ -329,22 +344,64 @@ class ServiceTabPbgTask
 
     private function refreshToken()
     {
-        try {
+        $maxRetries = 3; // Maximum retry attempts
+        $attempt = 0; 
 
-            $newAuthToken = $this->service_token->refresh_token($this->user_refresh_token);
+        while ($attempt < $maxRetries) {
+            try {
+                $attempt++;
+                Log::info("Attempt $attempt: Refreshing token...");
 
-            $this->user_token = $newAuthToken['access'];
+                $newAuthToken = $this->service_token->refresh_token($this->user_refresh_token);
-            $this->user_refresh_token = $newAuthToken['refresh'];
 
-            if (!$this->user_token) {
+                if (!isset($newAuthToken['access']) || !isset($newAuthToken['refresh'])) {
-                Log::error("Token refresh failed: No token received.");
+                    throw new \Exception("Invalid refresh token response.");
-                throw new \Exception("Failed to refresh token.");
+                }
+
+                $this->user_token = $newAuthToken['access'];
+                $this->user_refresh_token = $newAuthToken['refresh'];
+
+                Log::info("Token refreshed successfully on attempt $attempt.");
+                return; // Exit function on success
+            } catch (\Exception $e) {
+                Log::error("Token refresh failed on attempt $attempt: " . $e->getMessage());
+
+                if ($attempt >= $maxRetries) {
+                    Log::info("Max retries reached. Attempting to log in again...");
+                    break;
+                }
+
+                sleep(30); // Wait for 30 seconds before retrying
             }
+        }
 
-            Log::info("Token refreshed successfully.");
+        // If refresh fails after retries, attempt re-login
-        } catch (\Exception $e) {
+        $attempt = 0;
-            Log::error("Token refresh error: " . $e->getMessage());
+        while ($attempt < $maxRetries) {
-            throw new \Exception("Token refresh failed.");
+            try {
+                $attempt++;
+                Log::info("Attempt $attempt: Re-logging in...");
+
+                $loginAgain = $this->service_token->get_token(); // Login again
+
+                if (!isset($loginAgain['access']) || !isset($loginAgain['refresh'])) {
+                    throw new \Exception("Invalid login response.");
+                }
+
+                $this->user_token = $loginAgain['access'];
+                $this->user_refresh_token = $loginAgain['refresh'];
+
+                Log::info("Re-login successful on attempt $attempt.");
+                return; // Exit function on success
+            } catch (\Exception $e) {
+                Log::error("Re-login failed on attempt $attempt: " . $e->getMessage());
+
+                if ($attempt >= $maxRetries) {
+                    throw new \Exception("Both token refresh and login failed after $maxRetries attempts. " . $e->getMessage());
+                }
+
+                sleep(30); // Wait for 30 seconds before retrying
+            }
         }
     }
 

database/migrations/2025_03_21_162343_add_duration_in_import_datasources.php

@@ -0,0 +1,29 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        Schema::table('import_datasources', function (Blueprint $table) {
+            $table->timestamp('start_time')->nullable();
+            $table->timestamp('finish_time')->nullable();
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::table('import_datasources', function (Blueprint $table) {
+            $table->dropColumn(['start_time', 'finish_time']);
+        });
+    }
+};

deploy.sh

@@ -29,7 +29,11 @@ echo "🔄 Restarting PHP service..."
 systemctl reload $PHP_VERSION-fpm
 
 echo "🔁 Restarting Supervisor queue workers..."
-supervisorctl reload
+php artisan queue:restart
+
+supervisorctl reread
+supervisorctl update
+supervisorctl restart all
 
 php artisan up
 echo "🚀 Deployment completed successfully!"

resources/js/data/google-sheet/index.js

@@ -41,8 +41,8 @@ class GoogleSheets {
         tableContainer.innerHTML = "";
 
         // Get user permissions from data attributes
-        let canUpdate = tableContainer.getAttribute("data-updater") === "1";
+        // let canUpdate = tableContainer.getAttribute("data-updater") === "1";
-        let canDelete = tableContainer.getAttribute("data-destroyer") === "1";
+        // let canDelete = tableContainer.getAttribute("data-destroyer") === "1";
 
         this.table = new Grid({
             columns: [
@@ -65,25 +65,25 @@ class GoogleSheets {
                           </a>
                         `;
 
-                        if (canUpdate) {
+                        // if (canUpdate) {
-                            buttons += `
+                        //     buttons += `
-                            <a href="#" class="btn btn-yellow btn-sm d-inline-flex align-items-center justify-content-center">
+                        //     <a href="#" class="btn btn-yellow btn-sm d-inline-flex align-items-center justify-content-center">
-                                <i class='bx bx-edit'></i>
+                        //         <i class='bx bx-edit'></i>
-                            </a>
+                        //     </a>
-                        `;
+                        // `;
-                        }
+                        // }
 
-                        if (canDelete) {
+                        // if (canDelete) {
-                            buttons += `
+                        //     buttons += `
-                            <button data-id="${cell}" class="btn btn-sm btn-red btn-delete-google-sheet d-inline-flex align-items-center justify-content-center">
+                        //     <button data-id="${cell}" class="btn btn-sm btn-red btn-delete-google-sheet d-inline-flex align-items-center justify-content-center">
-                                <i class='bx bxs-trash'></i>
+                        //         <i class='bx bxs-trash'></i>
-                            </button>
+                        //     </button>
-                        `;
+                        // `;
-                        }
+                        // }
 
-                        if (!canUpdate && !canDelete) {
+                        // if (!canUpdate && !canDelete) {
-                            buttons = `<span class="text-muted">No Privilege</span>`;
+                        //     buttons = `<span class="text-muted">No Privilege</span>`;
-                        }
+                        // }
 
                         return gridjs.html(
                             `<div class="d-flex justify-content-center gap-2">${buttons}</div>`

resources/js/menus/index.js

@@ -39,7 +39,7 @@ class Menus {
                 "Name",
                 "Url",
                 "Icon",
-                "ParentID",
+                "Parent Name",
                 "Sort Order",
                 {
                     name: "Action",
@@ -97,16 +97,22 @@ class Menus {
                         .getAttribute("content")}`,
                     "Content-Type": "application/json",
                 },
-                then: (data) =>
+                then: (data) => {
-                    data.data.map((item) => [
+                    console.log("Full API Response:", data); // Log the full response
-                        item.id,
+
-                        item.name,
+                    return data.data.map((item, index) => {
-                        item.url,
+                        console.log(`Item ${index + 1}:`, item); // Log each item
-                        item.icon,
+                        return [
-                        item.parent_id,
+                            item.id,
-                        item.sort_order,
+                            item.name,
-                        item.id,
+                            item.url,
-                    ]),
+                            item.icon,
+                            item.parent?.name,
+                            item.sort_order,
+                            item.id,
+                        ];
+                    });
+                },
                 total: (data) => data.total,
             },
         }).render(tableContainer);

resources/js/settings/syncronize/syncronize.js

@@ -19,7 +19,16 @@ class SyncronizeTask {
             "table-import-datasources"
         );
         this.table = new gridjs.Grid({
-            columns: ["ID", "Message", "Response", "Status", "Created"],
+            columns: [
+                "ID",
+                "Message",
+                "Response",
+                "Status",
+                "Started",
+                "Duration",
+                "Finished",
+                "Created",
+            ],
             search: {
                 server: {
                     url: (prev, keyword) => `${prev}?search=${keyword}`,
@@ -49,6 +58,9 @@ class SyncronizeTask {
                         item.message,
                         item.response_body,
                         item.status,
+                        item.start_time,
+                        item.duration,
+                        item.finish_time,
                         item.created_at,
                     ]),
                 total: (data) => data.meta.total,
@@ -148,8 +160,11 @@ class SyncronizeTask {
             })
             .catch((err) => {
                 console.error("Fetch error:", err);
-                alert("An error occurred during synchronization" + err.message);
+                this.toastMessage.innerText =
+                    err.message || "Failed to syncronize something wrong!";
+                this.toast.show();
                 button.disabled = false;
+                spinner.classList.add("d-none");
             });
     }
 }

resources/views/auth/signin.blade.php

@@ -30,7 +30,7 @@ class="authentication-bg"
 
                             @if (sizeof($errors) > 0)
                             @foreach ($errors->all() as $error)
-                            <p class="text-red-600 mb-3">{{ $error }}</p>
+                            <p class="text-red mb-3">{{ $error }}</p>
                             @endforeach
                             @endif
 
@@ -39,6 +39,7 @@ class="authentication-bg"
                                 <input type="email" class="form-control" id="email" name="email" placeholder="Enter your email">
                             </div>
                             <div class="mb-3">
+                                <label for="password" class="form-label">Password</label>
                                 <input type="password" class="form-control" id="password" name="password" placeholder="Enter your password">
                             </div>
                             <div class="d-grid">

resources/views/data/google-sheet/index.blade.php

@@ -15,9 +15,9 @@
 		<div class="card w-100">
 			<div class="card-body">
 				<div class="d-flex flex-wrap justify-content-end align-items-center mb-2">
-					@if ($user_menu_permission['allow_create'])
+					<!-- @if ($user_menu_permission['allow_create'])
 						<a href="#" class="btn btn-success btn-sm d-block d-sm-inline w-auto">Create</a>
-					@endif
+					@endif -->
 				</div>
 				<div id="table-data-google-sheets"
                      data-updater="{{ $user_menu_permission['allow_update'] }}"