From f9e1aa1604d2ad60b4e1b1d9bceb04e109b0b9ca Mon Sep 17 00:00:00 2001 From: arifal Date: Tue, 18 Mar 2025 06:42:42 +0700 Subject: [PATCH] fix service scraping data --- app/Console/Commands/ScrapingData.php | 11 +- app/Services/ServicePbgTask.php | 145 +++++++- app/Services/ServiceTabPbgTask.php | 351 ++++++++++++++++++ ...74508_modify_email_on_task_assignments.php | 40 ++ 4 files changed, 526 insertions(+), 21 deletions(-) create mode 100644 app/Services/ServiceTabPbgTask.php create mode 100644 database/migrations/2025_03_17_174508_modify_email_on_task_assignments.php diff --git a/app/Console/Commands/ScrapingData.php b/app/Console/Commands/ScrapingData.php index b9b3614..2970378 100644 --- a/app/Console/Commands/ScrapingData.php +++ b/app/Console/Commands/ScrapingData.php @@ -5,6 +5,7 @@ namespace App\Console\Commands; use App\Models\ImportDatasource; use App\Services\ServiceGoogleSheet; use App\Services\ServicePbgTask; +use App\Services\ServiceTabPbgTask; use GuzzleHttp\Client; // Import Guzzle Client use Illuminate\Console\Command; use Illuminate\Support\Facades\Log; @@ -28,15 +29,17 @@ class ScrapingData extends Command private $client; private $service_pbg_task; + private $service_tab_pbg_task; /** * Inject dependencies. */ - public function __construct(Client $client, ServicePbgTask $service_pbg_task) + public function __construct(Client $client, ServicePbgTask $service_pbg_task, ServiceTabPbgTask $serviceTabPbgTask) { parent::__construct(); $this->client = $client; $this->service_pbg_task = $service_pbg_task; + $this->service_tab_pbg_task = $serviceTabPbgTask; } /** @@ -44,7 +47,6 @@ class ScrapingData extends Command */ public function handle() { - DB::beginTransaction(); // Start transaction try { // Create a record with "processing" status @@ -61,15 +63,16 @@ class ScrapingData extends Command // Run the ServicePbgTask with injected Guzzle Client $this->service_pbg_task->run_service(); + // run the service pbg task assignments + $this->service_tab_pbg_task->run_service(); + // Update the record status to "success" after completion $import_datasource->update([ 'status' => 'success', 'message' => 'Scraping completed successfully.' ]); - DB::commit(); // Commit the transaction if everything is successful } catch (\Exception $e) { - DB::rollBack(); // Rollback transaction on error // Log the error for debugging Log::error('Scraping failed: ' . $e->getMessage(), ['trace' => $e->getTraceAsString()]); diff --git a/app/Services/ServicePbgTask.php b/app/Services/ServicePbgTask.php index 2c0e513..08e5be5 100644 --- a/app/Services/ServicePbgTask.php +++ b/app/Services/ServicePbgTask.php @@ -3,6 +3,8 @@ namespace App\Services; use App\Models\GlobalSetting; +use App\Models\PbgTask; +use Carbon\Carbon; use GuzzleHttp\Client; use Exception; use Illuminate\Support\Facades\Log; @@ -14,6 +16,8 @@ class ServicePbgTask private $fetch_per_page; private $pbg_task_url; private $service_token; + private $user_token; + private $user_refresh_token; public function __construct(Client $client, ServiceTokenSIMBG $service_token) { @@ -21,42 +25,149 @@ class ServicePbgTask ->pluck('value', 'key'); $this->simbg_host = trim((string) ($settings['SIMBG_HOST'] ?? "")); - // $this->fetch_per_page = trim((string) ($settings['FETCH_PER_PAGE'] ?? "10")); - $this->fetch_per_page = 10; + $this->fetch_per_page = trim((string) ($settings['FETCH_PER_PAGE'] ?? "10")); $this->client = $client; $this->service_token = $service_token; $this->pbg_task_url = "{$this->simbg_host}/api/pbg/v1/list/?page=1&size={$this->fetch_per_page}&sort=ASC"; + $auth_data = $this->service_token->get_token(); + $this->user_token = $auth_data['access']; + $this->user_refresh_token = $auth_data['refresh']; } public function run_service() { - $this->fetch_pbg_task(); + try{ + $this->fetch_pbg_task(); + }catch(Exception $e){ + throw $e; + } } private function fetch_pbg_task() { try { - $token = $this->service_token->get_token(); + $currentPage = 1; + $totalPage = 1; + $options = [ 'headers' => [ - 'Authorization' => "Bearer {$token['access']}", - 'Accept' => 'application/json' + 'Authorization' => "Bearer {$this->user_token}", + 'Content-Type' => 'application/json' ] ]; - $fetch_data = $this->client->get($this->pbg_task_url, $options); - $response_status_code = $fetch_data->getStatusCode(); - $response = json_decode($fetch_data->getBody()->getContents(), true); - $data = $response['data']; - $total_page = $response['total_page']; + $maxRetries = 3; // Maximum number of retries + $initialDelay = 1; // Initial delay in seconds - Log::info("Successfully fetched PBG tasks", ['data' => $data]); - Log::info("Status code response", ['code' => $response_status_code]); - Log::info("Success", ['total_page' => $total_page]); - return $response; + $fetchData = function ($url) use (&$options, $maxRetries, $initialDelay) { + $retryCount = 0; + + while ($retryCount < $maxRetries) { + try { + return $this->client->get($url, $options); + } catch (\GuzzleHttp\Exception\ClientException $e) { + if ($e->getCode() === 401) { + Log::warning("Unauthorized. Refreshing token..."); + + // Refresh token + $auth_data = $this->service_token->refresh_token($this->user_refresh_token); + if (!isset($auth_data['access'])) { + Log::error("Token refresh failed."); + throw new Exception("Token refresh failed."); + } + + // Update tokens + $this->user_token = $auth_data['access']; + $this->user_refresh_token = $auth_data['refresh']; + + // Update headers + $options['headers']['Authorization'] = "Bearer {$this->user_token}"; + + // Retry request + return $this->client->get($url, $options); + } + throw $e; + } catch (\GuzzleHttp\Exception\ServerException | \GuzzleHttp\Exception\ConnectException $e) { + // Handle 502 or connection issues + if ($e->getCode() === 502) { + Log::warning("502 Bad Gateway - Retrying in {$initialDelay} seconds..."); + } else { + Log::error("Network error - Retrying in {$initialDelay} seconds..."); + } + + $retryCount++; + sleep($initialDelay); + $initialDelay *= 2; // Exponential backoff + } + } + + Log::error("Max retries reached. Failing request."); + throw new Exception("Max retries reached. Failing request."); + }; + + do { + $url = "{$this->simbg_host}/api/pbg/v1/list/?page={$currentPage}&size={$this->fetch_per_page}&sort=ASC"; + + $fetch_data = $fetchData($url); + if (!$fetch_data) { + Log::error("Failed to fetch data on page {$currentPage} after retries."); + throw new Exception("Failed to fetch data on page {$currentPage} after retries."); + } + + $response = json_decode($fetch_data->getBody()->getContents(), true); + if (!isset($response['data'])) { + Log::error("Invalid API response on page {$currentPage}"); + throw new Exception("Invalid API response on page {$currentPage}"); + } + + $data = $response['data']; + $totalPage = isset($response['total_page']) ? (int) $response['total_page'] : 1; + + $saved_data = []; + foreach ($data as $item) { + $saved_data[] = [ + 'uuid' => $item['uid'] ?? null, + 'name' => $item['name'] ?? null, + 'owner_name' => $item['owner_name'] ?? null, + 'application_type' => $item['application_type'] ?? null, + 'application_type_name' => $item['application_type_name'] ?? null, + 'condition' => $item['condition'] ?? null, + 'registration_number' => $item['registration_number'] ?? null, + 'document_number' => $item['document_number'] ?? null, + 'address' => $item['address'] ?? null, + 'status' => $item['status'] ?? null, + 'status_name' => $item['status_name'] ?? null, + 'slf_status' => $item['slf_status'] ?? null, + 'slf_status_name' => $item['slf_status_name'] ?? null, + 'function_type' => $item['function_type'] ?? null, + 'consultation_type' => $item['consultation_type'] ?? null, + 'due_date' => $item['due_date'] ?? null, + 'land_certificate_phase' => $item['land_certificate_phase'] ?? null, + 'task_created_at' => isset($item['created_at']) ? Carbon::parse($item['created_at'])->format('Y-m-d H:i:s') : null, + 'updated_at' => now(), + 'created_at' => now(), + ]; + } + + if (!empty($saved_data)) { + PbgTask::upsert($saved_data, ['uuid'], [ + 'name', 'owner_name', 'application_type', 'application_type_name', 'condition', + 'registration_number', 'document_number', 'address', 'status', 'status_name', + 'slf_status', 'slf_status_name', 'function_type', 'consultation_type', 'due_date', + 'land_certificate_phase', 'task_created_at', 'updated_at' + ]); + } + + Log::info("Page {$currentPage} fetched & saved", ['records' => count($saved_data)]); + + $currentPage++; + } while ($currentPage <= $totalPage); + + return true; } catch (Exception $e) { - Log::error("Failed to fetch PBG tasks", ['error' => $e->getMessage()]); - return null; // Return null if an error occurs + Log::error("Error fetching PBG tasks", ['error' => $e->getMessage()]); + throw $e; } } + } diff --git a/app/Services/ServiceTabPbgTask.php b/app/Services/ServiceTabPbgTask.php new file mode 100644 index 0000000..86895e6 --- /dev/null +++ b/app/Services/ServiceTabPbgTask.php @@ -0,0 +1,351 @@ +pluck('value', 'key'); + + $this->simbg_host = trim((string) ($settings['SIMBG_HOST'] ?? "")); + $this->fetch_per_page = trim((string) ($settings['FETCH_PER_PAGE'] ?? "10")); + $this->client = $client; + $this->service_token = $service_token; + $auth_data = $this->service_token->get_token(); + $this->user_token = $auth_data['access']; + $this->user_refresh_token = $auth_data['refresh']; + } + + public function run_service() + { + try { + $pbg_tasks = PbgTask::all(); + + foreach ($pbg_tasks as $pbg_task) { + $this->scraping_task_assignments($pbg_task->uuid); + $this->scraping_task_retributions($pbg_task->uuid); + $this->scraping_task_integrations($pbg_task->uuid); + + // Process task assignments here if needed + Log::info("Successfully fetched for UUID: {$pbg_task->uuid}"); + } + } catch (\Exception $e) { + Log::error("Failed to scrape task assignments: " . $e->getMessage()); + throw $e; + } + } + + private function scraping_task_assignments($uuid) + { + $url = "{$this->simbg_host}/api/pbg/v1/list-tim-penilai/{$uuid}/?page=1&size=10"; + $options = [ + 'headers' => [ + 'Authorization' => "Bearer {$this->user_token}", + 'Content-Type' => 'application/json' + ] + ]; + + $maxRetries = 3; + $initialDelay = 1; + $retriedAfter401 = false; + + for ($retryCount = 0; $retryCount < $maxRetries; $retryCount++) { + try { + $response = $this->client->get($url, $options); + $responseData = json_decode($response->getBody()->getContents(), true); + + if (empty($responseData['data']) || !is_array($responseData['data'])) { + return true; + } + + $task_assignments = []; + + foreach ($responseData['data'] as $data) { + $task_assignments[] = [ + 'pbg_task_uid' => $uuid, + 'user_id' => $data['user_id'] ?? null, + 'name' => $data['name'] ?? null, + 'username' => $data['username'] ?? null, + 'email' => $data['email'] ?? null, + 'phone_number' => $data['phone_number'] ?? null, + 'role' => $data['role'] ?? null, + 'role_name' => $data['role_name'] ?? null, + 'is_active' => $data['is_active'] ?? false, + 'file' => !empty($data['file']) ? json_encode($data['file']) : null, + 'expertise' => !empty($data['expertise']) ? json_encode($data['expertise']) : null, + 'experience' => !empty($data['experience']) ? json_encode($data['experience']) : null, + 'is_verif' => $data['is_verif'] ?? false, + 'uid' => $data['uid'] ?? null, + 'status' => $data['status'] ?? null, + 'status_name' => $data['status_name'] ?? null, + 'note' => $data['note'] ?? null, + 'ta_id' => $data['id'] ?? null, + 'created_at' => now(), + 'updated_at' => now(), + ]; + } + + if (!empty($task_assignments)) { + TaskAssignment::upsert( + $task_assignments, + ['uid'], + ['ta_id', 'name', 'username', 'email', 'phone_number', 'role', 'role_name', 'is_active', 'file', 'expertise', 'experience', 'is_verif', 'status', 'status_name', 'note', 'updated_at'] + ); + } + + return $responseData; + } catch (\GuzzleHttp\Exception\ClientException $e) { + if ($e->getCode() === 401 && !$retriedAfter401) { + Log::warning("401 Unauthorized - Refreshing token and retrying..."); + $this->refreshToken(); + $options['headers']['Authorization'] = "Bearer {$this->user_token}"; + $retriedAfter401 = true; + continue; // Retry with new token + } + + throw $e; + } catch (\GuzzleHttp\Exception\ServerException | \GuzzleHttp\Exception\ConnectException $e) { + if ($e->getCode() === 502) { + Log::warning("502 Bad Gateway - Retrying in {$initialDelay} seconds..."); + } else { + Log::error("Network error ({$e->getCode()}) - Retrying in {$initialDelay} seconds..."); + } + + sleep($initialDelay); + $initialDelay *= 2; + } catch (\Exception $e) { + Log::error("Unexpected error: " . $e->getMessage()); + throw $e; + } + } + + Log::error("Failed to fetch task assignments for UUID {$uuid} after {$maxRetries} retries."); + throw new \Exception("Failed to fetch task assignments for UUID {$uuid} after retries."); + } + + private function scraping_task_retributions($uuid) + { + $url = "{$this->simbg_host}/api/pbg/v1/detail/" . $uuid . "/retribution/submit/"; + $options = [ + 'headers' => [ + 'Authorization' => "Bearer {$this->user_token}", + 'Content-Type' => 'application/json' + ] + ]; + + $maxRetries = 3; + $initialDelay = 1; + $retriedAfter401 = false; + + for ($retryCount = 0; $retryCount < $maxRetries; $retryCount++) { + try { + $response = $this->client->get($url, $options); + $responseData = json_decode($response->getBody()->getContents(), true, 512, JSON_THROW_ON_ERROR); + + if (empty($responseData['data']) || !is_array($responseData['data'])) { + return true; + } + + $data = $responseData['data']; + + $detailCreatedAt = isset($data['created_at']) + ? Carbon::parse($data['created_at'])->format('Y-m-d H:i:s') + : null; + + $detailUpdatedAt = isset($data['updated_at']) + ? Carbon::parse($data['updated_at'])->format('Y-m-d H:i:s') + : null; + + $pbg_task_retributions = PbgTaskRetributions::updateOrCreate( + ['detail_id' => $data['id']], + [ + 'detail_uid' => $data['uid'] ?? null, + 'detail_created_at' => $detailCreatedAt ?? null, + 'detail_updated_at' => $detailUpdatedAt ?? null, + 'luas_bangunan' => $data['luas_bangunan'] ?? null, + 'indeks_lokalitas' => $data['indeks_lokalitas'] ?? null, + 'wilayah_shst' => $data['wilayah_shst'] ?? null, + 'kegiatan_id' => $data['kegiatan']['id'] ?? null, + 'kegiatan_name' => $data['kegiatan']['name'] ?? null, + 'nilai_shst' => $data['nilai_shst'] ?? null, + 'indeks_terintegrasi' => $data['indeks_terintegrasi'] ?? null, + 'indeks_bg_terbangun' => $data['indeks_bg_terbangun'] ?? null, + 'nilai_retribusi_bangunan' => $data['nilai_retribusi_bangunan'] ?? null, + 'nilai_prasarana' => $data['nilai_prasarana'] ?? null, + 'created_by' => $data['created_by'] ?? null, + 'pbg_document' => $data['pbg_document'] ?? null, + 'underpayment' => $data['underpayment'] ?? null, + 'skrd_amount' => $data['skrd_amount'] ?? null, + 'pbg_task_uid' => $uuid, + ] + ); + + $pbg_task_retribution_id = $pbg_task_retributions->id; + + $prasaranaData = $data['prasarana'] ?? []; + if (!empty($prasaranaData)) { + $insertData = array_map(fn($item) => [ + 'pbg_task_uid' => $uuid, + 'pbg_task_retribution_id' => $pbg_task_retribution_id, + 'prasarana_id' => $item['id'] ?? null, + 'prasarana_type' => $item['prasarana_type'] ?? null, + 'building_type' => $item['building_type'] ?? null, + 'total' => $item['total'] ?? null, + 'quantity' => $item['quantity'] ?? null, + 'unit' => $item['unit'] ?? null, + 'index_prasarana' => $item['index_prasarana'] ?? null, + ], $prasaranaData); + + PbgTaskPrasarana::upsert($insertData, ['prasarana_id']); + } + + return $responseData; + } catch (\GuzzleHttp\Exception\ClientException $e) { + if ($e->getCode() === 401 && !$retriedAfter401) { + Log::warning("401 Unauthorized - Refreshing token and retrying..."); + $this->refreshToken(); + $options['headers']['Authorization'] = "Bearer {$this->user_token}"; + $retriedAfter401 = true; + continue; + } + + return false; + } catch (\GuzzleHttp\Exception\ServerException | \GuzzleHttp\Exception\ConnectException $e) { + if ($e->getCode() === 502) { + Log::warning("502 Bad Gateway - Retrying in {$initialDelay} seconds..."); + } else { + Log::error("Network error ({$e->getCode()}) - Retrying in {$initialDelay} seconds..."); + } + + sleep($initialDelay); + $initialDelay *= 2; + } catch (\GuzzleHttp\Exception\RequestException $e) { + Log::error("Request error ({$e->getCode()}): " . $e->getMessage()); + return false; + } catch (\JsonException $e) { + Log::error("JSON decoding error: " . $e->getMessage()); + return false; + } catch (\Throwable $e) { + Log::critical("Unhandled error: " . $e->getMessage(), ['trace' => $e->getTraceAsString()]); + return false; + } + } + + Log::error("Failed to fetch task retributions for UUID {$uuid} after retries."); + throw new \Exception("Failed to fetch task retributions for UUID {$uuid} after retries."); + } + + private function scraping_task_integrations($uuid){ + $url = "{$this->simbg_host}/api/pbg/v1/detail/" . $uuid . "/retribution/indeks-terintegrasi/"; + $options = [ + 'headers' => [ + 'Authorization' => "Bearer {$this->user_token}", + 'Content-Type' => 'application/json' + ] + ]; + $maxRetries = 3; + $initialDelay = 1; + $retriedAfter401 = false; + for ($retryCount = 0; $retryCount < $maxRetries; $retryCount++) { + try { + $response = $this->client->get($url, $options); + $responseData = json_decode($response->getBody()->getContents(), true, 512, JSON_THROW_ON_ERROR); + + if (empty($responseData['data']) || !is_array($responseData['data'])) { + return true; + } + + $data = $responseData['data']; + + $integrations[] = [ + 'pbg_task_uid' => $uuid, + 'indeks_fungsi_bangunan' => $data['indeks_fungsi_bangunan'] ?? null, + 'indeks_parameter_kompleksitas' => $data['indeks_parameter_kompleksitas'] ?? null, + 'indeks_parameter_permanensi' => $data['indeks_parameter_permanensi'] ?? null, + 'indeks_parameter_ketinggian' => $data['indeks_parameter_ketinggian'] ?? null, + 'faktor_kepemilikan' => $data['faktor_kepemilikan'] ?? null, + 'indeks_terintegrasi' => $data['indeks_terintegrasi'] ?? null, + 'total' => $data['total'] ?? null, + ]; + + if (!empty($integrations)) { + PbgTaskIndexIntegrations::upsert($integrations, ['pbg_task_uid'], ['indeks_fungsi_bangunan', + 'indeks_parameter_kompleksitas', 'indeks_parameter_permanensi', 'indeks_parameter_ketinggian', 'faktor_kepemilikan', 'indeks_terintegrasi', 'total']); + } + + return $responseData; + } catch (\GuzzleHttp\Exception\ClientException $e) { + if ($e->getCode() === 401 && !$retriedAfter401) { + Log::warning("401 Unauthorized - Refreshing token and retrying..."); + $this->refreshToken(); + $options['headers']['Authorization'] = "Bearer {$this->user_token}"; + $retriedAfter401 = true; + continue; + } + + return false; + } catch (\GuzzleHttp\Exception\ServerException | \GuzzleHttp\Exception\ConnectException $e) { + if ($e->getCode() === 502) { + Log::warning("502 Bad Gateway - Retrying in {$initialDelay} seconds..."); + } else { + Log::error("Network error ({$e->getCode()}) - Retrying in {$initialDelay} seconds..."); + } + + sleep($initialDelay); + $initialDelay *= 2; + } catch (\GuzzleHttp\Exception\RequestException $e) { + Log::error("Request error ({$e->getCode()}): " . $e->getMessage()); + return false; + } catch (\JsonException $e) { + Log::error("JSON decoding error: " . $e->getMessage()); + return false; + } catch (\Throwable $e) { + Log::critical("Unhandled error: " . $e->getMessage(), ['trace' => $e->getTraceAsString()]); + return false; + } + } + + Log::error("Failed to fetch task index integration for UUID {$uuid} after retries."); + throw new \Exception("Failed to fetch task index integration for UUID {$uuid} after retries."); + } + + private function refreshToken() + { + try { + + $newAuthToken = $this->service_token->refresh_token($this->user_refresh_token); + + $this->user_token = $newAuthToken['access']; + $this->user_refresh_token = $newAuthToken['refresh']; + + if (!$this->user_token) { + Log::error("Token refresh failed: No token received."); + throw new \Exception("Failed to refresh token."); + } + + Log::info("Token refreshed successfully."); + } catch (\Exception $e) { + Log::error("Token refresh error: " . $e->getMessage()); + throw new \Exception("Token refresh failed."); + } + } + +} diff --git a/database/migrations/2025_03_17_174508_modify_email_on_task_assignments.php b/database/migrations/2025_03_17_174508_modify_email_on_task_assignments.php new file mode 100644 index 0000000..2d7b3ab --- /dev/null +++ b/database/migrations/2025_03_17_174508_modify_email_on_task_assignments.php @@ -0,0 +1,40 @@ +dropUnique('task_assignments_email_unique'); + } + + $indexes = DB::select("SHOW INDEXES FROM task_assignments WHERE Key_name = 'task_assignments_username_unique'"); + + if (!empty($indexes)) { + $table->dropUnique('task_assignments_username_unique'); + } + $table->string('email')->nullable()->change(); + $table->string('username')->nullable()->change(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('task_assignments', function (Blueprint $table) { + // + }); + } +};