add scheduler execute scraping data daily midnight
This commit is contained in:
34
app/Console/Commands/ExecuteScraping.php
Normal file
34
app/Console/Commands/ExecuteScraping.php
Normal file
@@ -0,0 +1,34 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
namespace App\Console\Commands;
|
||||||
|
|
||||||
|
use App\ServiceSIMBG;
|
||||||
|
use Illuminate\Console\Command;
|
||||||
|
use \Illuminate\Support\Facades\Log;
|
||||||
|
|
||||||
|
class ExecuteScraping extends Command
|
||||||
|
{
|
||||||
|
/**
|
||||||
|
* The name and signature of the console command.
|
||||||
|
*
|
||||||
|
* @var string
|
||||||
|
*/
|
||||||
|
protected $signature = 'app:execute-scraping';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The console command description.
|
||||||
|
*
|
||||||
|
* @var string
|
||||||
|
*/
|
||||||
|
protected $description = 'Execure scraping service daily every 12 pm';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Execute the console command.
|
||||||
|
*/
|
||||||
|
public function handle()
|
||||||
|
{
|
||||||
|
Log::info("running scheduler daily scraping");
|
||||||
|
$service = new ServiceSIMBG();
|
||||||
|
$service->syncTaskList();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -3,6 +3,7 @@
|
|||||||
namespace App\Providers;
|
namespace App\Providers;
|
||||||
|
|
||||||
use Illuminate\Support\ServiceProvider;
|
use Illuminate\Support\ServiceProvider;
|
||||||
|
use Carbon\Carbon;
|
||||||
|
|
||||||
class AppServiceProvider extends ServiceProvider
|
class AppServiceProvider extends ServiceProvider
|
||||||
{
|
{
|
||||||
@@ -19,6 +20,6 @@ class AppServiceProvider extends ServiceProvider
|
|||||||
*/
|
*/
|
||||||
public function boot(): void
|
public function boot(): void
|
||||||
{
|
{
|
||||||
//
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,6 +10,8 @@ use App\Models\PbgTaskPrasarana;
|
|||||||
use App\Models\PbgTaskRetributions;
|
use App\Models\PbgTaskRetributions;
|
||||||
use Exception;
|
use Exception;
|
||||||
use App\Models\PbgTask;
|
use App\Models\PbgTask;
|
||||||
|
use Illuminate\Support\Facades\Log;
|
||||||
|
|
||||||
class ServiceSIMBG
|
class ServiceSIMBG
|
||||||
{
|
{
|
||||||
private $email;
|
private $email;
|
||||||
@@ -33,7 +35,7 @@ class ServiceSIMBG
|
|||||||
|
|
||||||
$res = $clientHelper->post($url, $body);
|
$res = $clientHelper->post($url, $body);
|
||||||
if(!$res->original['success']){
|
if(!$res->original['success']){
|
||||||
\Log::error("Token not retrieved ", ['response' => $res]);
|
Log::error("Token not retrieved ", ['response' => $res]);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
return $res;
|
return $res;
|
||||||
@@ -47,7 +49,7 @@ class ServiceSIMBG
|
|||||||
|
|
||||||
if (!isset($resToken) || empty($resToken->original['data']['token']['access'])) {
|
if (!isset($resToken) || empty($resToken->original['data']['token']['access'])) {
|
||||||
// Log error
|
// Log error
|
||||||
\Log::error("Token not retrieved for syncIndexIntegration", ['uuid' => $uuid]);
|
Log::error("Token not retrieved for syncIndexIntegration", ['uuid' => $uuid]);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -60,13 +62,13 @@ class ServiceSIMBG
|
|||||||
|
|
||||||
if (empty($res->original['success']) || !$res->original['success']) {
|
if (empty($res->original['success']) || !$res->original['success']) {
|
||||||
// Log error
|
// Log error
|
||||||
\Log::error("API response indicates failure", ['url' => $url, 'uuid' => $uuid]);
|
Log::error("API response indicates failure", ['url' => $url, 'uuid' => $uuid]);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
$data = $res->original['data']['data'] ?? null;
|
$data = $res->original['data']['data'] ?? null;
|
||||||
if (!$data) {
|
if (!$data) {
|
||||||
\Log::error("No valid data returned from API", ['url' => $url, 'uuid' => $uuid]);
|
Log::error("No valid data returned from API", ['url' => $url, 'uuid' => $uuid]);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -84,7 +86,7 @@ class ServiceSIMBG
|
|||||||
);
|
);
|
||||||
|
|
||||||
// Log success
|
// Log success
|
||||||
\Log::info("syncIndexIntegration completed successfully", ['uuid' => $uuid]);
|
Log::info("syncIndexIntegration completed successfully", ['uuid' => $uuid]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -99,7 +101,7 @@ class ServiceSIMBG
|
|||||||
]);
|
]);
|
||||||
|
|
||||||
if (!isset($resToken) || empty($resToken->original['data']['token']['access'])) {
|
if (!isset($resToken) || empty($resToken->original['data']['token']['access'])) {
|
||||||
\Log::error("Token not retrieved for syncTaskList");
|
Log::error("Token not retrieved for syncTaskList");
|
||||||
$importDatasource->update([
|
$importDatasource->update([
|
||||||
'status' => ImportDatasourceStatus::Failed->value,
|
'status' => ImportDatasourceStatus::Failed->value,
|
||||||
'message' => 'Failed to retrive token'
|
'message' => 'Failed to retrive token'
|
||||||
@@ -120,7 +122,7 @@ class ServiceSIMBG
|
|||||||
|
|
||||||
$initialResponse = $clientHelper->get($url, $headers);
|
$initialResponse = $clientHelper->get($url, $headers);
|
||||||
if (empty($initialResponse->original['data']['total_page'])) {
|
if (empty($initialResponse->original['data']['total_page'])) {
|
||||||
\Log::error("Invalid response: no total_page", ['response' => $initialResponse->original]);
|
Log::error("Invalid response: no total_page", ['response' => $initialResponse->original]);
|
||||||
$importDatasource->update([
|
$importDatasource->update([
|
||||||
'status' => ImportDatasourceStatus::Failed->value,
|
'status' => ImportDatasourceStatus::Failed->value,
|
||||||
'message' => 'Invalid response: no total_page'
|
'message' => 'Invalid response: no total_page'
|
||||||
@@ -144,7 +146,7 @@ class ServiceSIMBG
|
|||||||
$response = $clientHelper->get($url, $headers);
|
$response = $clientHelper->get($url, $headers);
|
||||||
|
|
||||||
if (empty($response->original['data']['data'])) {
|
if (empty($response->original['data']['data'])) {
|
||||||
\Log::warning("No data found on page", ['page' => $currentPage]);
|
Log::warning("No data found on page", ['page' => $currentPage]);
|
||||||
$importDatasource->update([
|
$importDatasource->update([
|
||||||
'status' => ImportDatasourceStatus::Success->value,
|
'status' => ImportDatasourceStatus::Success->value,
|
||||||
'message' => 'Success but no data loaded on page'
|
'message' => 'Success but no data loaded on page'
|
||||||
@@ -186,7 +188,7 @@ class ServiceSIMBG
|
|||||||
$this->syncTaskDetailSubmit($item['uid']);
|
$this->syncTaskDetailSubmit($item['uid']);
|
||||||
$savedCount++;
|
$savedCount++;
|
||||||
} catch (Exception $e) {
|
} catch (Exception $e) {
|
||||||
\Log::error("Failed to process task", [
|
Log::error("Failed to process task", [
|
||||||
'error' => $e->getMessage(),
|
'error' => $e->getMessage(),
|
||||||
'task' => $item,
|
'task' => $item,
|
||||||
]);
|
]);
|
||||||
@@ -211,7 +213,7 @@ class ServiceSIMBG
|
|||||||
'message' => "Successfully success data: " .$savedCount. " failed data : " .$failedCount
|
'message' => "Successfully success data: " .$savedCount. " failed data : " .$failedCount
|
||||||
]);
|
]);
|
||||||
|
|
||||||
\Log::info("syncTaskList completed", $result);
|
Log::info("syncTaskList completed", $result);
|
||||||
return ApiResponse::successResponse(json_encode($result), "Successfully saved");
|
return ApiResponse::successResponse(json_encode($result), "Successfully saved");
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -223,7 +225,7 @@ class ServiceSIMBG
|
|||||||
|
|
||||||
if (!isset($resToken) || empty($resToken->original['data']['token']['access'])) {
|
if (!isset($resToken) || empty($resToken->original['data']['token']['access'])) {
|
||||||
// Log error
|
// Log error
|
||||||
\Log::error("Token not retrieved for syncTaskDetailSubmit");
|
Log::error("Token not retrieved for syncTaskDetailSubmit");
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -237,13 +239,13 @@ class ServiceSIMBG
|
|||||||
|
|
||||||
if (empty($res->original['success']) || !$res->original['success']) {
|
if (empty($res->original['success']) || !$res->original['success']) {
|
||||||
// Log error
|
// Log error
|
||||||
\Log::error("API response indicates failure", ['url' => $url, 'uuid' => $uuid]);
|
Log::error("API response indicates failure", ['url' => $url, 'uuid' => $uuid]);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
$data = $res->original['data']['data'] ?? [];
|
$data = $res->original['data']['data'] ?? [];
|
||||||
if (empty($data)) {
|
if (empty($data)) {
|
||||||
\Log::error("No data returned from API", ['url' => $url, 'uuid' => $uuid]);
|
Log::error("No data returned from API", ['url' => $url, 'uuid' => $uuid]);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -300,7 +302,7 @@ class ServiceSIMBG
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
\Log::info("syncTaskDetailSubmit completed successfully", ['uuid' => $uuid]);
|
Log::info("syncTaskDetailSubmit completed successfully", ['uuid' => $uuid]);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
2
package-lock.json
generated
2
package-lock.json
generated
@@ -1,5 +1,5 @@
|
|||||||
{
|
{
|
||||||
"name": "Darkone-Laravel",
|
"name": "dashboard-pupr",
|
||||||
"lockfileVersion": 3,
|
"lockfileVersion": 3,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"packages": {
|
"packages": {
|
||||||
|
|||||||
@@ -1,8 +1,11 @@
|
|||||||
<?php
|
<?php
|
||||||
|
|
||||||
|
use Illuminate\Support\Facades\Schedule;
|
||||||
use Illuminate\Foundation\Inspiring;
|
use Illuminate\Foundation\Inspiring;
|
||||||
use Illuminate\Support\Facades\Artisan;
|
use Illuminate\Support\Facades\Artisan;
|
||||||
|
|
||||||
Artisan::command('inspire', function () {
|
Artisan::command('inspire', function () {
|
||||||
$this->comment(Inspiring::quote());
|
$this->comment(Inspiring::quote());
|
||||||
})->purpose('Display an inspiring quote')->hourly();
|
})->purpose('Display an inspiring quote')->hourly();
|
||||||
|
|
||||||
|
Schedule::command("app:execute-scraping")->daily();
|
||||||
Reference in New Issue
Block a user