arifal/sibedas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix syncronize using worker and add duration syncronize

Browse Source
This commit is contained in:
arifal
2025-03-21 18:44:28 +07:00
parent 0a080763cd
commit 654d2efe19
11 changed files with 268 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

92
app/Console/Commands/ScrapingData.php
View File

@@ -2,10 +2,12 @@
namespace App\Console\Commands;
use App\Jobs\ScrapingDataJob;
use App\Models\ImportDatasource;
use App\Services\ServiceGoogleSheet;
use App\Services\ServicePbgTask;
use App\Services\ServiceTabPbgTask;
use App\Services\ServiceTokenSIMBG;
use GuzzleHttp\Client; // Import Guzzle Client
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Log;
@@ -26,64 +28,68 @@ class ScrapingData extends Command
* @var string
*/
protected $description = 'Command description';
private $client;
private $service_pbg_task;
private $service_tab_pbg_task;
/**
* Inject dependencies.
*/
public function __construct(Client $client, ServicePbgTask $service_pbg_task, ServiceTabPbgTask $serviceTabPbgTask)
{
public function __construct(
) {
parent::__construct();
$this->client = $client;
$this->service_pbg_task = $service_pbg_task;
$this->service_tab_pbg_task = $serviceTabPbgTask;
}
public function handle()
{
dispatch(new ScrapingDataJob());
$this->info("Scraping job dispatched successfully");
}
/**
* Execute the console command.
*/
public function handle()
{
// public function handle()
// {
try {
// Create a record with "processing" status
$import_datasource = ImportDatasource::create([
'message' => 'Initiating scraping...',
'response_body' => null,
'status' => 'processing'
]);
// try {
// // Create a record with "processing" status
// $import_datasource = ImportDatasource::create([
// 'message' => 'Initiating scraping...',
// 'response_body' => null,
// 'status' => 'processing',
// 'start_time' => now()
// ]);
// Run the service
$service_google_sheet = new ServiceGoogleSheet();
$service_google_sheet->run_service();
// // Run the service
// $service_google_sheet = new ServiceGoogleSheet();
// $service_google_sheet->run_service();
// Run the ServicePbgTask with injected Guzzle Client
$this->service_pbg_task->run_service();
// // Run the ServicePbgTask with injected Guzzle Client
// $this->service_pbg_task->run_service();
// run the service pbg task assignments
$this->service_tab_pbg_task->run_service();
// // run the service pbg task assignments
// $this->service_tab_pbg_task->run_service();
// Update the record status to "success" after completion
$import_datasource->update([
'status' => 'success',
'message' => 'Scraping completed successfully.'
]);
// // Update the record status to "success" after completion
// $import_datasource->update([
// 'status' => 'success',
// 'message' => 'Scraping completed successfully.',
// 'finish_time' => now()
// ]);
} catch (\Exception $e) {
// } catch (\Exception $e) {
// // Log the error for debugging
// Log::error('Scraping failed: ' . $e->getMessage(), ['trace' => $e->getTraceAsString()]);
// // Handle errors by updating the status to "failed"
// if (isset($import_datasource)) {
// $import_datasource->update([
// 'status' => 'failed',
// 'response_body' => 'Error: ' . $e->getMessage(),
// 'finish_time' => now()
// ]);
// }
// }
// }
// Log the error for debugging
Log::error('Scraping failed: ' . $e->getMessage(), ['trace' => $e->getTraceAsString()]);
// Handle errors by updating the status to "failed"
if (isset($import_datasource)) {
$import_datasource->update([
'status' => 'failed',
'response_body' => 'Error: ' . $e->getMessage()
]);
}
}
}
}

11
app/Http/Controllers/Api/ScrapingController.php
View File

@@ -4,9 +4,15 @@ namespace App\Http\Controllers\Api;
use App\Enums\ImportDatasourceStatus;
use App\Http\Controllers\Controller;
use App\Jobs\ScrapingDataJob;
use App\Jobs\SyncronizeSIMBG;
use App\Models\ImportDatasource;
use App\Traits\GlobalApiResponse;
use App\Services\ServiceTokenSIMBG;
use GuzzleHttp\Client;
use App\Services\ServiceGoogleSheet;
use App\Services\ServicePbgTask;
use App\Services\ServiceTabPbgTask;
use Illuminate\Support\Facades\Artisan;
use Illuminate\Http\Request;
@@ -23,8 +29,11 @@ class ScrapingController extends Controller
return $this->resError("Failed to execute while processing another scraping");
}
// use ole schema synchronization
// dispatch(new SyncronizeSIMBG());
Artisan::call("app:scraping-data");
// use new schema synchronization
dispatch(new ScrapingDataJob());
return $this->resSuccess(["message" => "Success execute scraping service on background, check status for more"]);
}

8
app/Http/Resources/ImportDatasourceResource.php
View File

@@ -2,6 +2,7 @@
namespace App\Http\Resources;
use Carbon\Carbon;
use Illuminate\Http\Request;
use Illuminate\Http\Resources\Json\JsonResource;
@@ -14,11 +15,18 @@ class ImportDatasourceResource extends JsonResource
*/
public function toArray(Request $request): array
{
$startTime = $this->start_time ? Carbon::parse($this->start_time) : null;
$finishTime = $this->finish_time ? Carbon::parse($this->finish_time) : null;
return [
"id"=> $this->id,
"message" => $this->message,
"response_body" => $this->response_body,
"status" => $this->status,
"start_time" => $startTime ? $startTime->toDateTimeString() : null,
"duration" => ($startTime && $finishTime)
? $finishTime->diff($startTime)->format('%H:%I:%S')
: null,
"finish_time" => $finishTime ? $finishTime->toDateTimeString() : null,
"created_at" => $this->created_at->toDateTimeString(),
"updated_at" => $this->updated_at->toDateTimeString(),
];

77
app/Jobs/ScrapingDataJob.php Normal file
View File

@@ -0,0 +1,77 @@
<?php
namespace App\Jobs;
use App\Models\ImportDatasource;
use App\Services\ServiceGoogleSheet;
use App\Services\ServicePbgTask;
use App\Services\ServiceTabPbgTask;
use App\Services\ServiceTokenSIMBG;
use GuzzleHttp\Client;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
class ScrapingDataJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
/**
* Inject dependencies instead of creating them inside.
*/
public function __construct(
) {
}
/**
* Execute the job.
*/
public function handle()
{
try {
$client = app(Client::class);
$service_pbg_task = app(ServicePbgTask::class);
$service_tab_pbg_task = app(ServiceTabPbgTask::class);
$service_google_sheet = app(ServiceGoogleSheet::class);
$service_token = app(ServiceTokenSIMBG::class);
// Create a record with "processing" status
$import_datasource = ImportDatasource::create([
'message' => 'Initiating scraping...',
'response_body' => null,
'status' => 'processing',
'start_time' => now()
]);
// Run the scraping services
$service_google_sheet->run_service();
$service_pbg_task->run_service();
$service_tab_pbg_task->run_service();
// Update status to success
$import_datasource->update([
'status' => 'success',
'message' => 'Scraping completed successfully.',
'finish_time' => now()
]);
} catch (\Exception $e) {
Log::error('Scraping failed: ' . $e->getMessage(), ['trace' => $e->getTraceAsString()]);
// Update status to failed
if (isset($import_datasource)) {
$import_datasource->update([
'status' => 'failed',
'response_body' => 'Error: ' . $e->getMessage(),
'finish_time' => now()
]);
}
// Mark the job as failed
$this->fail($e);
}
}
}

4
app/Models/ImportDatasource.php
View File

@@ -13,6 +13,8 @@ class ImportDatasource extends Model
'id',
'message',
'response_body',
'status'
'status',
'start_time',
'finish_time'
];
}

26
app/Providers/AppServiceProvider.php
View File

@@ -3,8 +3,13 @@
namespace App\Providers;
use App\Models\Menu;
use App\Services\ServiceGoogleSheet;
use App\Services\ServicePbgTask;
use App\Services\ServiceTabPbgTask;
use App\Services\ServiceTokenSIMBG;
use App\View\Components\Circle;
use Auth;
use GuzzleHttp\Client;
use Illuminate\Support\Facades\Blade;
use Illuminate\Support\Facades\View;
use Illuminate\Support\ServiceProvider;
@@ -19,11 +24,24 @@ class AppServiceProvider extends ServiceProvider
*/
public function register(): void
{
$this->app->singleton(GoogleSheetService::class, function () {
return new GoogleSheetService();
$this->app->bind(Client::class, function () {
return new Client();
});
$this->app->singleton(ServiceSIMBG::class, function ($app) {
return new ServiceSIMBG($app->make(GoogleSheetService::class));
$this->app->bind(ServiceTokenSIMBG::class, function ($app) {
return new ServiceTokenSIMBG();
});
$this->app->bind(ServicePbgTask::class, function ($app) {
return new ServicePbgTask($app->make(Client::class), $app->make(ServiceTokenSIMBG::class));
});
$this->app->bind(ServiceTabPbgTask::class, function ($app) {
return new ServiceTabPbgTask($app->make(Client::class), $app->make(ServiceTokenSIMBG::class));
});
$this->app->bind(ServiceGoogleSheet::class, function ($app) {
return new ServiceGoogleSheet();
});
}

63
app/Services/ServiceTabPbgTask.php
View File

@@ -344,22 +344,44 @@ class ServiceTabPbgTask
private function refreshToken()
{
try {
$newAuthToken = $this->service_token->refresh_token($this->user_refresh_token);
if (!isset($newAuthToken['access']) || !isset($newAuthToken['refresh'])) {
throw new \Exception("Invalid refresh token response.");
}
$this->user_token = $newAuthToken['access'];
$this->user_refresh_token = $newAuthToken['refresh'];
Log::info("Token refreshed successfully.");
} catch (\Exception $e) {
Log::error("Token refresh failed: " . $e->getMessage());
Log::info("Attempting to log in again...");
$maxRetries = 3; // Maximum retry attempts
$attempt = 0;
while ($attempt < $maxRetries) {
try {
$attempt++;
Log::info("Attempt $attempt: Refreshing token...");
$newAuthToken = $this->service_token->refresh_token($this->user_refresh_token);
if (!isset($newAuthToken['access']) || !isset($newAuthToken['refresh'])) {
throw new \Exception("Invalid refresh token response.");
}
$this->user_token = $newAuthToken['access'];
$this->user_refresh_token = $newAuthToken['refresh'];
Log::info("Token refreshed successfully on attempt $attempt.");
return; // Exit function on success
} catch (\Exception $e) {
Log::error("Token refresh failed on attempt $attempt: " . $e->getMessage());
if ($attempt >= $maxRetries) {
Log::info("Max retries reached. Attempting to log in again...");
break;
}
sleep(30); // Wait for 30 seconds before retrying
}
}
// If refresh fails after retries, attempt re-login
$attempt = 0;
while ($attempt < $maxRetries) {
try {
$attempt++;
Log::info("Attempt $attempt: Re-logging in...");
$loginAgain = $this->service_token->get_token(); // Login again
if (!isset($loginAgain['access']) || !isset($loginAgain['refresh'])) {
@@ -369,11 +391,18 @@ class ServiceTabPbgTask
$this->user_token = $loginAgain['access'];
$this->user_refresh_token = $loginAgain['refresh'];
Log::info("Re-login successful.");
Log::info("Re-login successful on attempt $attempt.");
return; // Exit function on success
} catch (\Exception $e) {
Log::error("Re-login failed: " . $e->getMessage());
throw new \Exception("Both token refresh and login failed.". $e->getMessage());
Log::error("Re-login failed on attempt $attempt: " . $e->getMessage());
if ($attempt >= $maxRetries) {
throw new \Exception("Both token refresh and login failed after $maxRetries attempts. " . $e->getMessage());
}
sleep(30); // Wait for 30 seconds before retrying
}
}
}
}