fix service scraping data

This commit is contained in:
arifal
2025-03-18 06:42:42 +07:00
parent 2e385f80cd
commit f9e1aa1604
4 changed files with 526 additions and 21 deletions

View File

@@ -5,6 +5,7 @@ namespace App\Console\Commands;
use App\Models\ImportDatasource; use App\Models\ImportDatasource;
use App\Services\ServiceGoogleSheet; use App\Services\ServiceGoogleSheet;
use App\Services\ServicePbgTask; use App\Services\ServicePbgTask;
use App\Services\ServiceTabPbgTask;
use GuzzleHttp\Client; // Import Guzzle Client use GuzzleHttp\Client; // Import Guzzle Client
use Illuminate\Console\Command; use Illuminate\Console\Command;
use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Log;
@@ -28,15 +29,17 @@ class ScrapingData extends Command
private $client; private $client;
private $service_pbg_task; private $service_pbg_task;
private $service_tab_pbg_task;
/** /**
* Inject dependencies. * Inject dependencies.
*/ */
public function __construct(Client $client, ServicePbgTask $service_pbg_task) public function __construct(Client $client, ServicePbgTask $service_pbg_task, ServiceTabPbgTask $serviceTabPbgTask)
{ {
parent::__construct(); parent::__construct();
$this->client = $client; $this->client = $client;
$this->service_pbg_task = $service_pbg_task; $this->service_pbg_task = $service_pbg_task;
$this->service_tab_pbg_task = $serviceTabPbgTask;
} }
/** /**
@@ -44,7 +47,6 @@ class ScrapingData extends Command
*/ */
public function handle() public function handle()
{ {
DB::beginTransaction(); // Start transaction
try { try {
// Create a record with "processing" status // Create a record with "processing" status
@@ -61,15 +63,16 @@ class ScrapingData extends Command
// Run the ServicePbgTask with injected Guzzle Client // Run the ServicePbgTask with injected Guzzle Client
$this->service_pbg_task->run_service(); $this->service_pbg_task->run_service();
// run the service pbg task assignments
$this->service_tab_pbg_task->run_service();
// Update the record status to "success" after completion // Update the record status to "success" after completion
$import_datasource->update([ $import_datasource->update([
'status' => 'success', 'status' => 'success',
'message' => 'Scraping completed successfully.' 'message' => 'Scraping completed successfully.'
]); ]);
DB::commit(); // Commit the transaction if everything is successful
} catch (\Exception $e) { } catch (\Exception $e) {
DB::rollBack(); // Rollback transaction on error
// Log the error for debugging // Log the error for debugging
Log::error('Scraping failed: ' . $e->getMessage(), ['trace' => $e->getTraceAsString()]); Log::error('Scraping failed: ' . $e->getMessage(), ['trace' => $e->getTraceAsString()]);

View File

@@ -3,6 +3,8 @@
namespace App\Services; namespace App\Services;
use App\Models\GlobalSetting; use App\Models\GlobalSetting;
use App\Models\PbgTask;
use Carbon\Carbon;
use GuzzleHttp\Client; use GuzzleHttp\Client;
use Exception; use Exception;
use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Log;
@@ -14,6 +16,8 @@ class ServicePbgTask
private $fetch_per_page; private $fetch_per_page;
private $pbg_task_url; private $pbg_task_url;
private $service_token; private $service_token;
private $user_token;
private $user_refresh_token;
public function __construct(Client $client, ServiceTokenSIMBG $service_token) public function __construct(Client $client, ServiceTokenSIMBG $service_token)
{ {
@@ -21,42 +25,149 @@ class ServicePbgTask
->pluck('value', 'key'); ->pluck('value', 'key');
$this->simbg_host = trim((string) ($settings['SIMBG_HOST'] ?? "")); $this->simbg_host = trim((string) ($settings['SIMBG_HOST'] ?? ""));
// $this->fetch_per_page = trim((string) ($settings['FETCH_PER_PAGE'] ?? "10")); $this->fetch_per_page = trim((string) ($settings['FETCH_PER_PAGE'] ?? "10"));
$this->fetch_per_page = 10;
$this->client = $client; $this->client = $client;
$this->service_token = $service_token; $this->service_token = $service_token;
$this->pbg_task_url = "{$this->simbg_host}/api/pbg/v1/list/?page=1&size={$this->fetch_per_page}&sort=ASC"; $this->pbg_task_url = "{$this->simbg_host}/api/pbg/v1/list/?page=1&size={$this->fetch_per_page}&sort=ASC";
$auth_data = $this->service_token->get_token();
$this->user_token = $auth_data['access'];
$this->user_refresh_token = $auth_data['refresh'];
} }
public function run_service() public function run_service()
{ {
try{
$this->fetch_pbg_task(); $this->fetch_pbg_task();
}catch(Exception $e){
throw $e;
}
} }
private function fetch_pbg_task() private function fetch_pbg_task()
{ {
try { try {
$token = $this->service_token->get_token(); $currentPage = 1;
$totalPage = 1;
$options = [ $options = [
'headers' => [ 'headers' => [
'Authorization' => "Bearer {$token['access']}", 'Authorization' => "Bearer {$this->user_token}",
'Accept' => 'application/json' 'Content-Type' => 'application/json'
] ]
]; ];
$fetch_data = $this->client->get($this->pbg_task_url, $options); $maxRetries = 3; // Maximum number of retries
$response_status_code = $fetch_data->getStatusCode(); $initialDelay = 1; // Initial delay in seconds
$response = json_decode($fetch_data->getBody()->getContents(), true);
$data = $response['data'];
$total_page = $response['total_page'];
Log::info("Successfully fetched PBG tasks", ['data' => $data]); $fetchData = function ($url) use (&$options, $maxRetries, $initialDelay) {
Log::info("Status code response", ['code' => $response_status_code]); $retryCount = 0;
Log::info("Success", ['total_page' => $total_page]);
return $response; while ($retryCount < $maxRetries) {
try {
return $this->client->get($url, $options);
} catch (\GuzzleHttp\Exception\ClientException $e) {
if ($e->getCode() === 401) {
Log::warning("Unauthorized. Refreshing token...");
// Refresh token
$auth_data = $this->service_token->refresh_token($this->user_refresh_token);
if (!isset($auth_data['access'])) {
Log::error("Token refresh failed.");
throw new Exception("Token refresh failed.");
}
// Update tokens
$this->user_token = $auth_data['access'];
$this->user_refresh_token = $auth_data['refresh'];
// Update headers
$options['headers']['Authorization'] = "Bearer {$this->user_token}";
// Retry request
return $this->client->get($url, $options);
}
throw $e;
} catch (\GuzzleHttp\Exception\ServerException | \GuzzleHttp\Exception\ConnectException $e) {
// Handle 502 or connection issues
if ($e->getCode() === 502) {
Log::warning("502 Bad Gateway - Retrying in {$initialDelay} seconds...");
} else {
Log::error("Network error - Retrying in {$initialDelay} seconds...");
}
$retryCount++;
sleep($initialDelay);
$initialDelay *= 2; // Exponential backoff
}
}
Log::error("Max retries reached. Failing request.");
throw new Exception("Max retries reached. Failing request.");
};
do {
$url = "{$this->simbg_host}/api/pbg/v1/list/?page={$currentPage}&size={$this->fetch_per_page}&sort=ASC";
$fetch_data = $fetchData($url);
if (!$fetch_data) {
Log::error("Failed to fetch data on page {$currentPage} after retries.");
throw new Exception("Failed to fetch data on page {$currentPage} after retries.");
}
$response = json_decode($fetch_data->getBody()->getContents(), true);
if (!isset($response['data'])) {
Log::error("Invalid API response on page {$currentPage}");
throw new Exception("Invalid API response on page {$currentPage}");
}
$data = $response['data'];
$totalPage = isset($response['total_page']) ? (int) $response['total_page'] : 1;
$saved_data = [];
foreach ($data as $item) {
$saved_data[] = [
'uuid' => $item['uid'] ?? null,
'name' => $item['name'] ?? null,
'owner_name' => $item['owner_name'] ?? null,
'application_type' => $item['application_type'] ?? null,
'application_type_name' => $item['application_type_name'] ?? null,
'condition' => $item['condition'] ?? null,
'registration_number' => $item['registration_number'] ?? null,
'document_number' => $item['document_number'] ?? null,
'address' => $item['address'] ?? null,
'status' => $item['status'] ?? null,
'status_name' => $item['status_name'] ?? null,
'slf_status' => $item['slf_status'] ?? null,
'slf_status_name' => $item['slf_status_name'] ?? null,
'function_type' => $item['function_type'] ?? null,
'consultation_type' => $item['consultation_type'] ?? null,
'due_date' => $item['due_date'] ?? null,
'land_certificate_phase' => $item['land_certificate_phase'] ?? null,
'task_created_at' => isset($item['created_at']) ? Carbon::parse($item['created_at'])->format('Y-m-d H:i:s') : null,
'updated_at' => now(),
'created_at' => now(),
];
}
if (!empty($saved_data)) {
PbgTask::upsert($saved_data, ['uuid'], [
'name', 'owner_name', 'application_type', 'application_type_name', 'condition',
'registration_number', 'document_number', 'address', 'status', 'status_name',
'slf_status', 'slf_status_name', 'function_type', 'consultation_type', 'due_date',
'land_certificate_phase', 'task_created_at', 'updated_at'
]);
}
Log::info("Page {$currentPage} fetched & saved", ['records' => count($saved_data)]);
$currentPage++;
} while ($currentPage <= $totalPage);
return true;
} catch (Exception $e) { } catch (Exception $e) {
Log::error("Failed to fetch PBG tasks", ['error' => $e->getMessage()]); Log::error("Error fetching PBG tasks", ['error' => $e->getMessage()]);
return null; // Return null if an error occurs throw $e;
} }
} }
} }

View File

@@ -0,0 +1,351 @@
<?php
namespace App\Services;
use App\Models\GlobalSetting;
use App\Models\PbgTask;
use App\Models\PbgTaskIndexIntegrations;
use App\Models\PbgTaskPrasarana;
use App\Models\PbgTaskRetributions;
use App\Models\TaskAssignment;
use Carbon\Carbon;
use GuzzleHttp\Client;
use Illuminate\Support\Facades\Log;
class ServiceTabPbgTask
{
private $client;
private $simbg_host;
private $fetch_per_page;
private $service_token;
private $user_token;
private $user_refresh_token;
public function __construct(Client $client, ServiceTokenSIMBG $service_token)
{
$settings = GlobalSetting::whereIn('key', ['SIMBG_HOST', 'FETCH_PER_PAGE'])
->pluck('value', 'key');
$this->simbg_host = trim((string) ($settings['SIMBG_HOST'] ?? ""));
$this->fetch_per_page = trim((string) ($settings['FETCH_PER_PAGE'] ?? "10"));
$this->client = $client;
$this->service_token = $service_token;
$auth_data = $this->service_token->get_token();
$this->user_token = $auth_data['access'];
$this->user_refresh_token = $auth_data['refresh'];
}
public function run_service()
{
try {
$pbg_tasks = PbgTask::all();
foreach ($pbg_tasks as $pbg_task) {
$this->scraping_task_assignments($pbg_task->uuid);
$this->scraping_task_retributions($pbg_task->uuid);
$this->scraping_task_integrations($pbg_task->uuid);
// Process task assignments here if needed
Log::info("Successfully fetched for UUID: {$pbg_task->uuid}");
}
} catch (\Exception $e) {
Log::error("Failed to scrape task assignments: " . $e->getMessage());
throw $e;
}
}
private function scraping_task_assignments($uuid)
{
$url = "{$this->simbg_host}/api/pbg/v1/list-tim-penilai/{$uuid}/?page=1&size=10";
$options = [
'headers' => [
'Authorization' => "Bearer {$this->user_token}",
'Content-Type' => 'application/json'
]
];
$maxRetries = 3;
$initialDelay = 1;
$retriedAfter401 = false;
for ($retryCount = 0; $retryCount < $maxRetries; $retryCount++) {
try {
$response = $this->client->get($url, $options);
$responseData = json_decode($response->getBody()->getContents(), true);
if (empty($responseData['data']) || !is_array($responseData['data'])) {
return true;
}
$task_assignments = [];
foreach ($responseData['data'] as $data) {
$task_assignments[] = [
'pbg_task_uid' => $uuid,
'user_id' => $data['user_id'] ?? null,
'name' => $data['name'] ?? null,
'username' => $data['username'] ?? null,
'email' => $data['email'] ?? null,
'phone_number' => $data['phone_number'] ?? null,
'role' => $data['role'] ?? null,
'role_name' => $data['role_name'] ?? null,
'is_active' => $data['is_active'] ?? false,
'file' => !empty($data['file']) ? json_encode($data['file']) : null,
'expertise' => !empty($data['expertise']) ? json_encode($data['expertise']) : null,
'experience' => !empty($data['experience']) ? json_encode($data['experience']) : null,
'is_verif' => $data['is_verif'] ?? false,
'uid' => $data['uid'] ?? null,
'status' => $data['status'] ?? null,
'status_name' => $data['status_name'] ?? null,
'note' => $data['note'] ?? null,
'ta_id' => $data['id'] ?? null,
'created_at' => now(),
'updated_at' => now(),
];
}
if (!empty($task_assignments)) {
TaskAssignment::upsert(
$task_assignments,
['uid'],
['ta_id', 'name', 'username', 'email', 'phone_number', 'role', 'role_name', 'is_active', 'file', 'expertise', 'experience', 'is_verif', 'status', 'status_name', 'note', 'updated_at']
);
}
return $responseData;
} catch (\GuzzleHttp\Exception\ClientException $e) {
if ($e->getCode() === 401 && !$retriedAfter401) {
Log::warning("401 Unauthorized - Refreshing token and retrying...");
$this->refreshToken();
$options['headers']['Authorization'] = "Bearer {$this->user_token}";
$retriedAfter401 = true;
continue; // Retry with new token
}
throw $e;
} catch (\GuzzleHttp\Exception\ServerException | \GuzzleHttp\Exception\ConnectException $e) {
if ($e->getCode() === 502) {
Log::warning("502 Bad Gateway - Retrying in {$initialDelay} seconds...");
} else {
Log::error("Network error ({$e->getCode()}) - Retrying in {$initialDelay} seconds...");
}
sleep($initialDelay);
$initialDelay *= 2;
} catch (\Exception $e) {
Log::error("Unexpected error: " . $e->getMessage());
throw $e;
}
}
Log::error("Failed to fetch task assignments for UUID {$uuid} after {$maxRetries} retries.");
throw new \Exception("Failed to fetch task assignments for UUID {$uuid} after retries.");
}
private function scraping_task_retributions($uuid)
{
$url = "{$this->simbg_host}/api/pbg/v1/detail/" . $uuid . "/retribution/submit/";
$options = [
'headers' => [
'Authorization' => "Bearer {$this->user_token}",
'Content-Type' => 'application/json'
]
];
$maxRetries = 3;
$initialDelay = 1;
$retriedAfter401 = false;
for ($retryCount = 0; $retryCount < $maxRetries; $retryCount++) {
try {
$response = $this->client->get($url, $options);
$responseData = json_decode($response->getBody()->getContents(), true, 512, JSON_THROW_ON_ERROR);
if (empty($responseData['data']) || !is_array($responseData['data'])) {
return true;
}
$data = $responseData['data'];
$detailCreatedAt = isset($data['created_at'])
? Carbon::parse($data['created_at'])->format('Y-m-d H:i:s')
: null;
$detailUpdatedAt = isset($data['updated_at'])
? Carbon::parse($data['updated_at'])->format('Y-m-d H:i:s')
: null;
$pbg_task_retributions = PbgTaskRetributions::updateOrCreate(
['detail_id' => $data['id']],
[
'detail_uid' => $data['uid'] ?? null,
'detail_created_at' => $detailCreatedAt ?? null,
'detail_updated_at' => $detailUpdatedAt ?? null,
'luas_bangunan' => $data['luas_bangunan'] ?? null,
'indeks_lokalitas' => $data['indeks_lokalitas'] ?? null,
'wilayah_shst' => $data['wilayah_shst'] ?? null,
'kegiatan_id' => $data['kegiatan']['id'] ?? null,
'kegiatan_name' => $data['kegiatan']['name'] ?? null,
'nilai_shst' => $data['nilai_shst'] ?? null,
'indeks_terintegrasi' => $data['indeks_terintegrasi'] ?? null,
'indeks_bg_terbangun' => $data['indeks_bg_terbangun'] ?? null,
'nilai_retribusi_bangunan' => $data['nilai_retribusi_bangunan'] ?? null,
'nilai_prasarana' => $data['nilai_prasarana'] ?? null,
'created_by' => $data['created_by'] ?? null,
'pbg_document' => $data['pbg_document'] ?? null,
'underpayment' => $data['underpayment'] ?? null,
'skrd_amount' => $data['skrd_amount'] ?? null,
'pbg_task_uid' => $uuid,
]
);
$pbg_task_retribution_id = $pbg_task_retributions->id;
$prasaranaData = $data['prasarana'] ?? [];
if (!empty($prasaranaData)) {
$insertData = array_map(fn($item) => [
'pbg_task_uid' => $uuid,
'pbg_task_retribution_id' => $pbg_task_retribution_id,
'prasarana_id' => $item['id'] ?? null,
'prasarana_type' => $item['prasarana_type'] ?? null,
'building_type' => $item['building_type'] ?? null,
'total' => $item['total'] ?? null,
'quantity' => $item['quantity'] ?? null,
'unit' => $item['unit'] ?? null,
'index_prasarana' => $item['index_prasarana'] ?? null,
], $prasaranaData);
PbgTaskPrasarana::upsert($insertData, ['prasarana_id']);
}
return $responseData;
} catch (\GuzzleHttp\Exception\ClientException $e) {
if ($e->getCode() === 401 && !$retriedAfter401) {
Log::warning("401 Unauthorized - Refreshing token and retrying...");
$this->refreshToken();
$options['headers']['Authorization'] = "Bearer {$this->user_token}";
$retriedAfter401 = true;
continue;
}
return false;
} catch (\GuzzleHttp\Exception\ServerException | \GuzzleHttp\Exception\ConnectException $e) {
if ($e->getCode() === 502) {
Log::warning("502 Bad Gateway - Retrying in {$initialDelay} seconds...");
} else {
Log::error("Network error ({$e->getCode()}) - Retrying in {$initialDelay} seconds...");
}
sleep($initialDelay);
$initialDelay *= 2;
} catch (\GuzzleHttp\Exception\RequestException $e) {
Log::error("Request error ({$e->getCode()}): " . $e->getMessage());
return false;
} catch (\JsonException $e) {
Log::error("JSON decoding error: " . $e->getMessage());
return false;
} catch (\Throwable $e) {
Log::critical("Unhandled error: " . $e->getMessage(), ['trace' => $e->getTraceAsString()]);
return false;
}
}
Log::error("Failed to fetch task retributions for UUID {$uuid} after retries.");
throw new \Exception("Failed to fetch task retributions for UUID {$uuid} after retries.");
}
private function scraping_task_integrations($uuid){
$url = "{$this->simbg_host}/api/pbg/v1/detail/" . $uuid . "/retribution/indeks-terintegrasi/";
$options = [
'headers' => [
'Authorization' => "Bearer {$this->user_token}",
'Content-Type' => 'application/json'
]
];
$maxRetries = 3;
$initialDelay = 1;
$retriedAfter401 = false;
for ($retryCount = 0; $retryCount < $maxRetries; $retryCount++) {
try {
$response = $this->client->get($url, $options);
$responseData = json_decode($response->getBody()->getContents(), true, 512, JSON_THROW_ON_ERROR);
if (empty($responseData['data']) || !is_array($responseData['data'])) {
return true;
}
$data = $responseData['data'];
$integrations[] = [
'pbg_task_uid' => $uuid,
'indeks_fungsi_bangunan' => $data['indeks_fungsi_bangunan'] ?? null,
'indeks_parameter_kompleksitas' => $data['indeks_parameter_kompleksitas'] ?? null,
'indeks_parameter_permanensi' => $data['indeks_parameter_permanensi'] ?? null,
'indeks_parameter_ketinggian' => $data['indeks_parameter_ketinggian'] ?? null,
'faktor_kepemilikan' => $data['faktor_kepemilikan'] ?? null,
'indeks_terintegrasi' => $data['indeks_terintegrasi'] ?? null,
'total' => $data['total'] ?? null,
];
if (!empty($integrations)) {
PbgTaskIndexIntegrations::upsert($integrations, ['pbg_task_uid'], ['indeks_fungsi_bangunan',
'indeks_parameter_kompleksitas', 'indeks_parameter_permanensi', 'indeks_parameter_ketinggian', 'faktor_kepemilikan', 'indeks_terintegrasi', 'total']);
}
return $responseData;
} catch (\GuzzleHttp\Exception\ClientException $e) {
if ($e->getCode() === 401 && !$retriedAfter401) {
Log::warning("401 Unauthorized - Refreshing token and retrying...");
$this->refreshToken();
$options['headers']['Authorization'] = "Bearer {$this->user_token}";
$retriedAfter401 = true;
continue;
}
return false;
} catch (\GuzzleHttp\Exception\ServerException | \GuzzleHttp\Exception\ConnectException $e) {
if ($e->getCode() === 502) {
Log::warning("502 Bad Gateway - Retrying in {$initialDelay} seconds...");
} else {
Log::error("Network error ({$e->getCode()}) - Retrying in {$initialDelay} seconds...");
}
sleep($initialDelay);
$initialDelay *= 2;
} catch (\GuzzleHttp\Exception\RequestException $e) {
Log::error("Request error ({$e->getCode()}): " . $e->getMessage());
return false;
} catch (\JsonException $e) {
Log::error("JSON decoding error: " . $e->getMessage());
return false;
} catch (\Throwable $e) {
Log::critical("Unhandled error: " . $e->getMessage(), ['trace' => $e->getTraceAsString()]);
return false;
}
}
Log::error("Failed to fetch task index integration for UUID {$uuid} after retries.");
throw new \Exception("Failed to fetch task index integration for UUID {$uuid} after retries.");
}
private function refreshToken()
{
try {
$newAuthToken = $this->service_token->refresh_token($this->user_refresh_token);
$this->user_token = $newAuthToken['access'];
$this->user_refresh_token = $newAuthToken['refresh'];
if (!$this->user_token) {
Log::error("Token refresh failed: No token received.");
throw new \Exception("Failed to refresh token.");
}
Log::info("Token refreshed successfully.");
} catch (\Exception $e) {
Log::error("Token refresh error: " . $e->getMessage());
throw new \Exception("Token refresh failed.");
}
}
}

View File

@@ -0,0 +1,40 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('task_assignments', function (Blueprint $table) {
$indexes = DB::select("SHOW INDEXES FROM task_assignments WHERE Key_name = 'task_assignments_email_unique'");
if (!empty($indexes)) {
$table->dropUnique('task_assignments_email_unique');
}
$indexes = DB::select("SHOW INDEXES FROM task_assignments WHERE Key_name = 'task_assignments_username_unique'");
if (!empty($indexes)) {
$table->dropUnique('task_assignments_username_unique');
}
$table->string('email')->nullable()->change();
$table->string('username')->nullable()->change();
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('task_assignments', function (Blueprint $table) {
//
});
}
};