diff --git a/README.md b/README.md index 2309142..7cf8829 100755 --- a/README.md +++ b/README.md @@ -17,10 +17,10 @@ sudo nano /etc/supervisor/conf.d/laravel-worker.conf [program:laravel-worker] process_name=%(program_name)s_%(process_num)02d -command=php /home/arifal/development/sibedas-pbg-web/artisan queue:work --queue=default --timeout=40000 --tries=1 --sleep=3 +command=php /home/arifal/development/sibedas-pbg-web/artisan queue:work --queue=default --timeout=40000 --tries=1 autostart=true autorestart=true -numprocs=4 +numprocs=1 redirect_stderr=true stdout_logfile=/home/arifal/development/sibedas-pbg-web/storage/logs/worker.log stopasgroup=true diff --git a/app/Console/Commands/ScrapingData.php b/app/Console/Commands/ScrapingData.php index ef7425e..3539cc6 100644 --- a/app/Console/Commands/ScrapingData.php +++ b/app/Console/Commands/ScrapingData.php @@ -2,10 +2,12 @@ namespace App\Console\Commands; +use App\Jobs\ScrapingDataJob; use App\Models\ImportDatasource; use App\Services\ServiceGoogleSheet; use App\Services\ServicePbgTask; use App\Services\ServiceTabPbgTask; +use App\Services\ServiceTokenSIMBG; use GuzzleHttp\Client; // Import Guzzle Client use Illuminate\Console\Command; use Illuminate\Support\Facades\Log; @@ -26,64 +28,68 @@ class ScrapingData extends Command * @var string */ protected $description = 'Command description'; - - private $client; - private $service_pbg_task; - private $service_tab_pbg_task; - /** * Inject dependencies. */ - public function __construct(Client $client, ServicePbgTask $service_pbg_task, ServiceTabPbgTask $serviceTabPbgTask) - { + public function __construct( + ) { parent::__construct(); - $this->client = $client; - $this->service_pbg_task = $service_pbg_task; - $this->service_tab_pbg_task = $serviceTabPbgTask; + } + + public function handle() + { + dispatch(new ScrapingDataJob()); + + $this->info("Scraping job dispatched successfully"); } /** * Execute the console command. */ - public function handle() - { + // public function handle() + // { - try { - // Create a record with "processing" status - $import_datasource = ImportDatasource::create([ - 'message' => 'Initiating scraping...', - 'response_body' => null, - 'status' => 'processing' - ]); + // try { + // // Create a record with "processing" status + // $import_datasource = ImportDatasource::create([ + // 'message' => 'Initiating scraping...', + // 'response_body' => null, + // 'status' => 'processing', + // 'start_time' => now() + // ]); - // Run the service - $service_google_sheet = new ServiceGoogleSheet(); - $service_google_sheet->run_service(); + // // Run the service + // $service_google_sheet = new ServiceGoogleSheet(); + // $service_google_sheet->run_service(); - // Run the ServicePbgTask with injected Guzzle Client - $this->service_pbg_task->run_service(); + // // Run the ServicePbgTask with injected Guzzle Client + // $this->service_pbg_task->run_service(); - // run the service pbg task assignments - $this->service_tab_pbg_task->run_service(); + // // run the service pbg task assignments + // $this->service_tab_pbg_task->run_service(); - // Update the record status to "success" after completion - $import_datasource->update([ - 'status' => 'success', - 'message' => 'Scraping completed successfully.' - ]); + // // Update the record status to "success" after completion + // $import_datasource->update([ + // 'status' => 'success', + // 'message' => 'Scraping completed successfully.', + // 'finish_time' => now() + // ]); - } catch (\Exception $e) { + // } catch (\Exception $e) { + + // // Log the error for debugging + // Log::error('Scraping failed: ' . $e->getMessage(), ['trace' => $e->getTraceAsString()]); + + // // Handle errors by updating the status to "failed" + // if (isset($import_datasource)) { + // $import_datasource->update([ + // 'status' => 'failed', + // 'response_body' => 'Error: ' . $e->getMessage(), + // 'finish_time' => now() + // ]); + // } + // } + // } - // Log the error for debugging - Log::error('Scraping failed: ' . $e->getMessage(), ['trace' => $e->getTraceAsString()]); - // Handle errors by updating the status to "failed" - if (isset($import_datasource)) { - $import_datasource->update([ - 'status' => 'failed', - 'response_body' => 'Error: ' . $e->getMessage() - ]); - } - } - } } diff --git a/app/Http/Controllers/Api/ScrapingController.php b/app/Http/Controllers/Api/ScrapingController.php index aeb574c..148265d 100644 --- a/app/Http/Controllers/Api/ScrapingController.php +++ b/app/Http/Controllers/Api/ScrapingController.php @@ -4,9 +4,15 @@ namespace App\Http\Controllers\Api; use App\Enums\ImportDatasourceStatus; use App\Http\Controllers\Controller; +use App\Jobs\ScrapingDataJob; use App\Jobs\SyncronizeSIMBG; use App\Models\ImportDatasource; use App\Traits\GlobalApiResponse; +use App\Services\ServiceTokenSIMBG; +use GuzzleHttp\Client; +use App\Services\ServiceGoogleSheet; +use App\Services\ServicePbgTask; +use App\Services\ServiceTabPbgTask; use Illuminate\Support\Facades\Artisan; use Illuminate\Http\Request; @@ -23,8 +29,11 @@ class ScrapingController extends Controller return $this->resError("Failed to execute while processing another scraping"); } + // use ole schema synchronization // dispatch(new SyncronizeSIMBG()); - Artisan::call("app:scraping-data"); + + // use new schema synchronization + dispatch(new ScrapingDataJob()); return $this->resSuccess(["message" => "Success execute scraping service on background, check status for more"]); } diff --git a/app/Http/Resources/ImportDatasourceResource.php b/app/Http/Resources/ImportDatasourceResource.php index a19e69f..bd8c7bc 100644 --- a/app/Http/Resources/ImportDatasourceResource.php +++ b/app/Http/Resources/ImportDatasourceResource.php @@ -2,6 +2,7 @@ namespace App\Http\Resources; +use Carbon\Carbon; use Illuminate\Http\Request; use Illuminate\Http\Resources\Json\JsonResource; @@ -14,11 +15,18 @@ class ImportDatasourceResource extends JsonResource */ public function toArray(Request $request): array { + $startTime = $this->start_time ? Carbon::parse($this->start_time) : null; + $finishTime = $this->finish_time ? Carbon::parse($this->finish_time) : null; return [ "id"=> $this->id, "message" => $this->message, "response_body" => $this->response_body, "status" => $this->status, + "start_time" => $startTime ? $startTime->toDateTimeString() : null, + "duration" => ($startTime && $finishTime) + ? $finishTime->diff($startTime)->format('%H:%I:%S') + : null, + "finish_time" => $finishTime ? $finishTime->toDateTimeString() : null, "created_at" => $this->created_at->toDateTimeString(), "updated_at" => $this->updated_at->toDateTimeString(), ]; diff --git a/app/Jobs/ScrapingDataJob.php b/app/Jobs/ScrapingDataJob.php new file mode 100644 index 0000000..d9cc039 --- /dev/null +++ b/app/Jobs/ScrapingDataJob.php @@ -0,0 +1,77 @@ + 'Initiating scraping...', + 'response_body' => null, + 'status' => 'processing', + 'start_time' => now() + ]); + + // Run the scraping services + $service_google_sheet->run_service(); + $service_pbg_task->run_service(); + $service_tab_pbg_task->run_service(); + + // Update status to success + $import_datasource->update([ + 'status' => 'success', + 'message' => 'Scraping completed successfully.', + 'finish_time' => now() + ]); + + } catch (\Exception $e) { + Log::error('Scraping failed: ' . $e->getMessage(), ['trace' => $e->getTraceAsString()]); + + // Update status to failed + if (isset($import_datasource)) { + $import_datasource->update([ + 'status' => 'failed', + 'response_body' => 'Error: ' . $e->getMessage(), + 'finish_time' => now() + ]); + } + + // Mark the job as failed + $this->fail($e); + } + } +} diff --git a/app/Models/ImportDatasource.php b/app/Models/ImportDatasource.php index 9bdb995..4fc6d12 100644 --- a/app/Models/ImportDatasource.php +++ b/app/Models/ImportDatasource.php @@ -13,6 +13,8 @@ class ImportDatasource extends Model 'id', 'message', 'response_body', - 'status' + 'status', + 'start_time', + 'finish_time' ]; } diff --git a/app/Providers/AppServiceProvider.php b/app/Providers/AppServiceProvider.php index 59cb8b6..2dd445b 100755 --- a/app/Providers/AppServiceProvider.php +++ b/app/Providers/AppServiceProvider.php @@ -3,8 +3,13 @@ namespace App\Providers; use App\Models\Menu; +use App\Services\ServiceGoogleSheet; +use App\Services\ServicePbgTask; +use App\Services\ServiceTabPbgTask; +use App\Services\ServiceTokenSIMBG; use App\View\Components\Circle; use Auth; +use GuzzleHttp\Client; use Illuminate\Support\Facades\Blade; use Illuminate\Support\Facades\View; use Illuminate\Support\ServiceProvider; @@ -19,11 +24,24 @@ class AppServiceProvider extends ServiceProvider */ public function register(): void { - $this->app->singleton(GoogleSheetService::class, function () { - return new GoogleSheetService(); + $this->app->bind(Client::class, function () { + return new Client(); }); - $this->app->singleton(ServiceSIMBG::class, function ($app) { - return new ServiceSIMBG($app->make(GoogleSheetService::class)); + + $this->app->bind(ServiceTokenSIMBG::class, function ($app) { + return new ServiceTokenSIMBG(); + }); + + $this->app->bind(ServicePbgTask::class, function ($app) { + return new ServicePbgTask($app->make(Client::class), $app->make(ServiceTokenSIMBG::class)); + }); + + $this->app->bind(ServiceTabPbgTask::class, function ($app) { + return new ServiceTabPbgTask($app->make(Client::class), $app->make(ServiceTokenSIMBG::class)); + }); + + $this->app->bind(ServiceGoogleSheet::class, function ($app) { + return new ServiceGoogleSheet(); }); } diff --git a/app/Services/ServiceTabPbgTask.php b/app/Services/ServiceTabPbgTask.php index 001b2ab..79229b7 100644 --- a/app/Services/ServiceTabPbgTask.php +++ b/app/Services/ServiceTabPbgTask.php @@ -344,22 +344,44 @@ class ServiceTabPbgTask private function refreshToken() { - try { - $newAuthToken = $this->service_token->refresh_token($this->user_refresh_token); - - if (!isset($newAuthToken['access']) || !isset($newAuthToken['refresh'])) { - throw new \Exception("Invalid refresh token response."); - } - - $this->user_token = $newAuthToken['access']; - $this->user_refresh_token = $newAuthToken['refresh']; - - Log::info("Token refreshed successfully."); - } catch (\Exception $e) { - Log::error("Token refresh failed: " . $e->getMessage()); - Log::info("Attempting to log in again..."); + $maxRetries = 3; // Maximum retry attempts + $attempt = 0; + while ($attempt < $maxRetries) { try { + $attempt++; + Log::info("Attempt $attempt: Refreshing token..."); + + $newAuthToken = $this->service_token->refresh_token($this->user_refresh_token); + + if (!isset($newAuthToken['access']) || !isset($newAuthToken['refresh'])) { + throw new \Exception("Invalid refresh token response."); + } + + $this->user_token = $newAuthToken['access']; + $this->user_refresh_token = $newAuthToken['refresh']; + + Log::info("Token refreshed successfully on attempt $attempt."); + return; // Exit function on success + } catch (\Exception $e) { + Log::error("Token refresh failed on attempt $attempt: " . $e->getMessage()); + + if ($attempt >= $maxRetries) { + Log::info("Max retries reached. Attempting to log in again..."); + break; + } + + sleep(30); // Wait for 30 seconds before retrying + } + } + + // If refresh fails after retries, attempt re-login + $attempt = 0; + while ($attempt < $maxRetries) { + try { + $attempt++; + Log::info("Attempt $attempt: Re-logging in..."); + $loginAgain = $this->service_token->get_token(); // Login again if (!isset($loginAgain['access']) || !isset($loginAgain['refresh'])) { @@ -369,11 +391,18 @@ class ServiceTabPbgTask $this->user_token = $loginAgain['access']; $this->user_refresh_token = $loginAgain['refresh']; - Log::info("Re-login successful."); + Log::info("Re-login successful on attempt $attempt."); + return; // Exit function on success } catch (\Exception $e) { - Log::error("Re-login failed: " . $e->getMessage()); - throw new \Exception("Both token refresh and login failed.". $e->getMessage()); + Log::error("Re-login failed on attempt $attempt: " . $e->getMessage()); + + if ($attempt >= $maxRetries) { + throw new \Exception("Both token refresh and login failed after $maxRetries attempts. " . $e->getMessage()); + } + + sleep(30); // Wait for 30 seconds before retrying } } } + } diff --git a/database/migrations/2025_03_21_162343_add_duration_in_import_datasources.php b/database/migrations/2025_03_21_162343_add_duration_in_import_datasources.php new file mode 100644 index 0000000..78ff9ff --- /dev/null +++ b/database/migrations/2025_03_21_162343_add_duration_in_import_datasources.php @@ -0,0 +1,29 @@ +timestamp('start_time')->nullable(); + $table->timestamp('finish_time')->nullable(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('import_datasources', function (Blueprint $table) { + $table->dropColumn(['start_time', 'finish_time']); + }); + } +}; diff --git a/deploy.sh b/deploy.sh index b30c8b2..42d5c6c 100755 --- a/deploy.sh +++ b/deploy.sh @@ -29,7 +29,11 @@ echo "🔄 Restarting PHP service..." systemctl reload $PHP_VERSION-fpm echo "🔁 Restarting Supervisor queue workers..." -supervisorctl reload +php artisan queue:restart + +supervisorctl reread +supervisorctl update +supervisorctl restart all php artisan up echo "🚀 Deployment completed successfully!" \ No newline at end of file diff --git a/resources/js/settings/syncronize/syncronize.js b/resources/js/settings/syncronize/syncronize.js index 7d62b55..d86f067 100644 --- a/resources/js/settings/syncronize/syncronize.js +++ b/resources/js/settings/syncronize/syncronize.js @@ -19,7 +19,16 @@ class SyncronizeTask { "table-import-datasources" ); this.table = new gridjs.Grid({ - columns: ["ID", "Message", "Response", "Status", "Created"], + columns: [ + "ID", + "Message", + "Response", + "Status", + "Started", + "Duration", + "Finished", + "Created", + ], search: { server: { url: (prev, keyword) => `${prev}?search=${keyword}`, @@ -49,6 +58,9 @@ class SyncronizeTask { item.message, item.response_body, item.status, + item.start_time, + item.duration, + item.finish_time, item.created_at, ]), total: (data) => data.meta.total, @@ -148,8 +160,11 @@ class SyncronizeTask { }) .catch((err) => { console.error("Fetch error:", err); - alert("An error occurred during synchronization" + err.message); + this.toastMessage.innerText = + err.message || "Failed to syncronize something wrong!"; + this.toast.show(); button.disabled = false; + spinner.classList.add("d-none"); }); } }