arifal/sibedas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix syncronize using worker and add duration syncronize

Browse Source
This commit is contained in:
arifal
2025-03-21 18:44:28 +07:00
parent 0a080763cd
commit 654d2efe19
11 changed files with 268 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@@ -17,10 +17,10 @@ sudo nano /etc/supervisor/conf.d/laravel-worker.conf
[program:laravel-worker]
process_name=%(program_name)s_%(process_num)02d
command=php /home/arifal/development/sibedas-pbg-web/artisan queue:work --queue=default --timeout=40000 --tries=1 --sleep=3
command=php /home/arifal/development/sibedas-pbg-web/artisan queue:work --queue=default --timeout=40000 --tries=1
autostart=true
autorestart=true
numprocs=4
numprocs=1
redirect_stderr=true
stdout_logfile=/home/arifal/development/sibedas-pbg-web/storage/logs/worker.log
stopasgroup=true

92
app/Console/Commands/ScrapingData.php
View File

@@ -2,10 +2,12 @@
namespace App\Console\Commands;
use App\Jobs\ScrapingDataJob;
use App\Models\ImportDatasource;
use App\Services\ServiceGoogleSheet;
use App\Services\ServicePbgTask;
use App\Services\ServiceTabPbgTask;
use App\Services\ServiceTokenSIMBG;
use GuzzleHttp\Client; // Import Guzzle Client
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Log;
@@ -26,64 +28,68 @@ class ScrapingData extends Command
* @var string
*/
protected $description = 'Command description';
private $client;
private $service_pbg_task;
private $service_tab_pbg_task;
/**
* Inject dependencies.
*/
public function __construct(Client $client, ServicePbgTask $service_pbg_task, ServiceTabPbgTask $serviceTabPbgTask)
{
public function __construct(
) {
parent::__construct();
$this->client = $client;
$this->service_pbg_task = $service_pbg_task;
$this->service_tab_pbg_task = $serviceTabPbgTask;
}
public function handle()
{
dispatch(new ScrapingDataJob());
$this->info("Scraping job dispatched successfully");
}
/**
* Execute the console command.
*/
public function handle()
{
// public function handle()
// {
try {
// Create a record with "processing" status
$import_datasource = ImportDatasource::create([
'message' => 'Initiating scraping...',
'response_body' => null,
'status' => 'processing'
]);
// try {
// // Create a record with "processing" status
// $import_datasource = ImportDatasource::create([
// 'message' => 'Initiating scraping...',
// 'response_body' => null,
// 'status' => 'processing',
// 'start_time' => now()
// ]);
// Run the service
$service_google_sheet = new ServiceGoogleSheet();
$service_google_sheet->run_service();
// // Run the service
// $service_google_sheet = new ServiceGoogleSheet();
// $service_google_sheet->run_service();
// Run the ServicePbgTask with injected Guzzle Client
$this->service_pbg_task->run_service();
// // Run the ServicePbgTask with injected Guzzle Client
// $this->service_pbg_task->run_service();
// run the service pbg task assignments
$this->service_tab_pbg_task->run_service();
// // run the service pbg task assignments
// $this->service_tab_pbg_task->run_service();
// Update the record status to "success" after completion
$import_datasource->update([
'status' => 'success',
'message' => 'Scraping completed successfully.'
]);
// // Update the record status to "success" after completion
// $import_datasource->update([
// 'status' => 'success',
// 'message' => 'Scraping completed successfully.',
// 'finish_time' => now()
// ]);
} catch (\Exception $e) {
// } catch (\Exception $e) {
// // Log the error for debugging
// Log::error('Scraping failed: ' . $e->getMessage(), ['trace' => $e->getTraceAsString()]);
// // Handle errors by updating the status to "failed"
// if (isset($import_datasource)) {
// $import_datasource->update([
// 'status' => 'failed',
// 'response_body' => 'Error: ' . $e->getMessage(),
// 'finish_time' => now()
// ]);
// }
// }
// }
// Log the error for debugging
Log::error('Scraping failed: ' . $e->getMessage(), ['trace' => $e->getTraceAsString()]);
// Handle errors by updating the status to "failed"
if (isset($import_datasource)) {
$import_datasource->update([
'status' => 'failed',
'response_body' => 'Error: ' . $e->getMessage()
]);
}
}
}
}

11
app/Http/Controllers/Api/ScrapingController.php
View File

@@ -4,9 +4,15 @@ namespace App\Http\Controllers\Api;
use App\Enums\ImportDatasourceStatus;
use App\Http\Controllers\Controller;
use App\Jobs\ScrapingDataJob;
use App\Jobs\SyncronizeSIMBG;
use App\Models\ImportDatasource;
use App\Traits\GlobalApiResponse;
use App\Services\ServiceTokenSIMBG;
use GuzzleHttp\Client;
use App\Services\ServiceGoogleSheet;
use App\Services\ServicePbgTask;
use App\Services\ServiceTabPbgTask;
use Illuminate\Support\Facades\Artisan;
use Illuminate\Http\Request;
@@ -23,8 +29,11 @@ class ScrapingController extends Controller
return $this->resError("Failed to execute while processing another scraping");
}
// use ole schema synchronization
// dispatch(new SyncronizeSIMBG());
Artisan::call("app:scraping-data");
// use new schema synchronization
dispatch(new ScrapingDataJob());
return $this->resSuccess(["message" => "Success execute scraping service on background, check status for more"]);
}

8
app/Http/Resources/ImportDatasourceResource.php
View File

@@ -2,6 +2,7 @@
namespace App\Http\Resources;
use Carbon\Carbon;
use Illuminate\Http\Request;
use Illuminate\Http\Resources\Json\JsonResource;
@@ -14,11 +15,18 @@ class ImportDatasourceResource extends JsonResource
*/
public function toArray(Request $request): array
{
$startTime = $this->start_time ? Carbon::parse($this->start_time) : null;
$finishTime = $this->finish_time ? Carbon::parse($this->finish_time) : null;
return [
"id"=> $this->id,
"message" => $this->message,
"response_body" => $this->response_body,
"status" => $this->status,
"start_time" => $startTime ? $startTime->toDateTimeString() : null,
"duration" => ($startTime && $finishTime)
? $finishTime->diff($startTime)->format('%H:%I:%S')
: null,
"finish_time" => $finishTime ? $finishTime->toDateTimeString() : null,
"created_at" => $this->created_at->toDateTimeString(),
"updated_at" => $this->updated_at->toDateTimeString(),
];

77
app/Jobs/ScrapingDataJob.php Normal file
View File

@@ -0,0 +1,77 @@
<?php
namespace App\Jobs;
use App\Models\ImportDatasource;
use App\Services\ServiceGoogleSheet;
use App\Services\ServicePbgTask;
use App\Services\ServiceTabPbgTask;
use App\Services\ServiceTokenSIMBG;
use GuzzleHttp\Client;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
class ScrapingDataJob implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
/**
* Inject dependencies instead of creating them inside.
*/
public function __construct(
) {
}
/**
* Execute the job.
*/
public function handle()
{
try {
$client = app(Client::class);
$service_pbg_task = app(ServicePbgTask::class);
$service_tab_pbg_task = app(ServiceTabPbgTask::class);
$service_google_sheet = app(ServiceGoogleSheet::class);
$service_token = app(ServiceTokenSIMBG::class);
// Create a record with "processing" status
$import_datasource = ImportDatasource::create([
'message' => 'Initiating scraping...',
'response_body' => null,
'status' => 'processing',
'start_time' => now()
]);
// Run the scraping services
$service_google_sheet->run_service();
$service_pbg_task->run_service();
$service_tab_pbg_task->run_service();
// Update status to success
$import_datasource->update([
'status' => 'success',
'message' => 'Scraping completed successfully.',
'finish_time' => now()
]);
} catch (\Exception $e) {
Log::error('Scraping failed: ' . $e->getMessage(), ['trace' => $e->getTraceAsString()]);
// Update status to failed
if (isset($import_datasource)) {
$import_datasource->update([
'status' => 'failed',
'response_body' => 'Error: ' . $e->getMessage(),
'finish_time' => now()
]);
}
// Mark the job as failed
$this->fail($e);
}
}
}

4
app/Models/ImportDatasource.php
View File

@@ -13,6 +13,8 @@ class ImportDatasource extends Model
'id',
'message',
'response_body',
'status'
'status',
'start_time',
'finish_time'
];
}

26
app/Providers/AppServiceProvider.php
View File

@@ -3,8 +3,13 @@
namespace App\Providers;
use App\Models\Menu;
use App\Services\ServiceGoogleSheet;
use App\Services\ServicePbgTask;
use App\Services\ServiceTabPbgTask;
use App\Services\ServiceTokenSIMBG;
use App\View\Components\Circle;
use Auth;
use GuzzleHttp\Client;
use Illuminate\Support\Facades\Blade;
use Illuminate\Support\Facades\View;
use Illuminate\Support\ServiceProvider;
@@ -19,11 +24,24 @@ class AppServiceProvider extends ServiceProvider
*/
public function register(): void
{
$this->app->singleton(GoogleSheetService::class, function () {
return new GoogleSheetService();
$this->app->bind(Client::class, function () {
return new Client();
});
$this->app->singleton(ServiceSIMBG::class, function ($app) {
return new ServiceSIMBG($app->make(GoogleSheetService::class));
$this->app->bind(ServiceTokenSIMBG::class, function ($app) {
return new ServiceTokenSIMBG();
});
$this->app->bind(ServicePbgTask::class, function ($app) {
return new ServicePbgTask($app->make(Client::class), $app->make(ServiceTokenSIMBG::class));
});
$this->app->bind(ServiceTabPbgTask::class, function ($app) {
return new ServiceTabPbgTask($app->make(Client::class), $app->make(ServiceTokenSIMBG::class));
});
$this->app->bind(ServiceGoogleSheet::class, function ($app) {
return new ServiceGoogleSheet();
});
}

63
app/Services/ServiceTabPbgTask.php
View File

@@ -344,22 +344,44 @@ class ServiceTabPbgTask
private function refreshToken()
{
try {
$newAuthToken = $this->service_token->refresh_token($this->user_refresh_token);
if (!isset($newAuthToken['access']) || !isset($newAuthToken['refresh'])) {
throw new \Exception("Invalid refresh token response.");
}
$this->user_token = $newAuthToken['access'];
$this->user_refresh_token = $newAuthToken['refresh'];
Log::info("Token refreshed successfully.");
} catch (\Exception $e) {
Log::error("Token refresh failed: " . $e->getMessage());
Log::info("Attempting to log in again...");
$maxRetries = 3; // Maximum retry attempts
$attempt = 0;
while ($attempt < $maxRetries) {
try {
$attempt++;
Log::info("Attempt $attempt: Refreshing token...");
$newAuthToken = $this->service_token->refresh_token($this->user_refresh_token);
if (!isset($newAuthToken['access']) || !isset($newAuthToken['refresh'])) {
throw new \Exception("Invalid refresh token response.");
}
$this->user_token = $newAuthToken['access'];
$this->user_refresh_token = $newAuthToken['refresh'];
Log::info("Token refreshed successfully on attempt $attempt.");
return; // Exit function on success
} catch (\Exception $e) {
Log::error("Token refresh failed on attempt $attempt: " . $e->getMessage());
if ($attempt >= $maxRetries) {
Log::info("Max retries reached. Attempting to log in again...");
break;
}
sleep(30); // Wait for 30 seconds before retrying
}
}
// If refresh fails after retries, attempt re-login
$attempt = 0;
while ($attempt < $maxRetries) {
try {
$attempt++;
Log::info("Attempt $attempt: Re-logging in...");
$loginAgain = $this->service_token->get_token(); // Login again
if (!isset($loginAgain['access']) || !isset($loginAgain['refresh'])) {
@@ -369,11 +391,18 @@ class ServiceTabPbgTask
$this->user_token = $loginAgain['access'];
$this->user_refresh_token = $loginAgain['refresh'];
Log::info("Re-login successful.");
Log::info("Re-login successful on attempt $attempt.");
return; // Exit function on success
} catch (\Exception $e) {
Log::error("Re-login failed: " . $e->getMessage());
throw new \Exception("Both token refresh and login failed.". $e->getMessage());
Log::error("Re-login failed on attempt $attempt: " . $e->getMessage());
if ($attempt >= $maxRetries) {
throw new \Exception("Both token refresh and login failed after $maxRetries attempts. " . $e->getMessage());
}
sleep(30); // Wait for 30 seconds before retrying
}
}
}
}

29
database/migrations/2025_03_21_162343_add_duration_in_import_datasources.php Normal file
View File

@@ -0,0 +1,29 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('import_datasources', function (Blueprint $table) {
$table->timestamp('start_time')->nullable();
$table->timestamp('finish_time')->nullable();
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('import_datasources', function (Blueprint $table) {
$table->dropColumn(['start_time', 'finish_time']);
});
}
};

6
deploy.sh
View File

@@ -29,7 +29,11 @@ echo "🔄 Restarting PHP service..."
systemctl reload $PHP_VERSION-fpm
echo "🔁 Restarting Supervisor queue workers..."
supervisorctl reload
php artisan queue:restart
supervisorctl reread
supervisorctl update
supervisorctl restart all
php artisan up
echo "🚀 Deployment completed successfully!"

19
resources/js/settings/syncronize/syncronize.js
View File

@@ -19,7 +19,16 @@ class SyncronizeTask {
"table-import-datasources"
);
this.table = new gridjs.Grid({
columns: ["ID", "Message", "Response", "Status", "Created"],
columns: [
"ID",
"Message",
"Response",
"Status",
"Started",
"Duration",
"Finished",
"Created",
],
search: {
server: {
url: (prev, keyword) => `${prev}?search=${keyword}`,
@@ -49,6 +58,9 @@ class SyncronizeTask {
item.message,
item.response_body,
item.status,
item.start_time,
item.duration,
item.finish_time,
item.created_at,
]),
total: (data) => data.meta.total,
@@ -148,8 +160,11 @@ class SyncronizeTask {
})
.catch((err) => {
console.error("Fetch error:", err);
alert("An error occurred during synchronization" + err.message);
this.toastMessage.innerText =
err.message || "Failed to syncronize something wrong!";
this.toast.show();
button.disabled = false;
spinner.classList.add("d-none");
});
}
}