arifal/sibedas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix query scraping using upsert and connect dashboard

Browse Source
This commit is contained in:
arifal
2025-01-30 22:57:15 +07:00
parent 1f6ef5f110
commit d70fefb12d
9 changed files with 1113 additions and 976 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
app/Http/Controllers/Api/DashboardController.php
View File

@@ -69,4 +69,29 @@ class DashboardController extends Controller
];
return $this->resSuccess($result);
}
public function pbgTaskDocuments(Request $request){
$request->validate([
'status' => 'required|string'
]);
$businessData = DB::table('pbg_task')
->leftJoin('pbg_task_retributions', 'pbg_task.uuid', '=', 'pbg_task_retributions.pbg_task_uid')
->select(
DB::raw('COUNT(DISTINCT pbg_task.id) as task_count'),
DB::raw('SUM(pbg_task_retributions.nilai_retribusi_bangunan) as total_retribution')
)
->where(function ($query) use ($request) {
$query->where("pbg_task.status", "=", $request->get('status'));
})
->first();
$taskCount = $businessData->task_count;
$taskTotal = $businessData->total_retribution;
$result = [
"count" => $taskCount,
"series" => [$taskCount],
"total" => $taskTotal
];
return $this->resSuccess($result);
}
}

177
app/ServiceSIMBG.php
View File

@@ -12,6 +12,7 @@ use Exception;
use App\Models\PbgTask;
use App\Traits\GlobalApiResponse;
use Illuminate\Support\Facades\Log;
use Carbon\Carbon;
class ServiceSIMBG
{
@@ -19,6 +20,7 @@ class ServiceSIMBG
private $email;
private $password;
private $simbg_host;
private $fetch_per_page;
/**
* Create a new class instance.
*/
@@ -27,6 +29,7 @@ class ServiceSIMBG
$this->email = trim((string) GlobalSetting::where('key','SIMBG_EMAIL')->first()->value);
$this->password = trim((string) GlobalSetting::where('key','SIMBG_PASSWORD')->first()->value);
$this->simbg_host = trim((string)GlobalSetting::where('key','SIMBG_HOST')->first()->value);
$this->fetch_per_page = trim((string)GlobalSetting::where('key','FETCH_PER_PAGE')->first()->value);
}
public function getToken(){
@@ -92,41 +95,32 @@ class ServiceSIMBG
// Log success
Log::info("syncIndexIntegration completed successfully", ['uuid' => $uuid]);
}
public function syncTaskList()
{
$clientHelper = new ServiceClient($this->simbg_host);
$resToken = $this->getToken();
// create log import datasource
$importDatasource = ImportDatasource::create([
'status' => ImportDatasourceStatus::Processing->value,
]);
if (!isset($resToken) || empty($resToken->original['data']['token']['access'])) {
Log::error("Token not retrieved for syncTaskList");
if (empty($resToken->original['data']['token']['access'])) {
$importDatasource->update([
'status' => ImportDatasourceStatus::Failed->value,
'message' => 'Failed to retrive token'
'message' => 'Failed to retrieve token'
]);
return $this->resError("Failed to retrive token");
return $this->resError("Failed to retrieve token");
}
$apiToken = $resToken->original['data']['token']['access'];
$queryParams = http_build_query([
'page' => 1,
'size' => 20,
'sort' => 'ASC',
// 'type' => 'task',
]);
$url = "/api/pbg/v1/list/?" . $queryParams;
$headers = ['Authorization' => "Bearer " . $apiToken];
$url = "/api/pbg/v1/list/?page=1&size={$this->fetch_per_page}&sort=ASC";
$initialResponse = $clientHelper->get($url, $headers);
if (empty($initialResponse->original['data']['total_page'])) {
Log::error("Invalid response: no total_page", ['response' => $initialResponse->original]);
$totalPage = $initialResponse->original['data']['total_page'] ?? 0;
if ($totalPage === 0) {
$importDatasource->update([
'status' => ImportDatasourceStatus::Failed->value,
'message' => 'Invalid response: no total_page'
@@ -134,97 +128,76 @@ class ServiceSIMBG
return $this->resError("Invalid response from API");
}
$totalPage = $initialResponse->original['data']['total_page'];
$savedCount = 0;
$failedCount = 0;
$savedCount = $failedCount = 0;
for ($currentPage = 1; $currentPage <= $totalPage; $currentPage++) {
$pageUrl = "/api/pbg/v1/list/?page={$currentPage}&size={$this->fetch_per_page}&sort=ASC";
$token = $this->getToken();
$simbg_token = $token->original['data']['token']['access'];
$headers = ['Authorization' => "Bearer " . $simbg_token];
$headers = ['Authorization' => "Bearer " . $token];
$response = $clientHelper->get($pageUrl, $headers);
$tasks = $response->original['data']['data'] ?? [];
$queryParams = http_build_query([
'page' => $currentPage,
'size' => 20,
'sort' => 'ASC',
// 'type' => 'task'
]);
$url = "/api/pbg/v1/list/?" . $queryParams;
$response = $clientHelper->get($url, $headers);
if (empty($response->original['data']['data'])) {
if (empty($tasks)) {
Log::warning("No data found on page", ['page' => $currentPage]);
$importDatasource->update([
'status' => ImportDatasourceStatus::Success->value,
'message' => 'Success but no data loaded on page'
]);
continue;
}
foreach ($response->original['data']['data'] as $item) {
try {
$taskCreatedAt = isset($item['created_at'])
? \Carbon\Carbon::parse($item['created_at'])->format('Y-m-d H:i:s')
: null;
PbgTask::updateOrCreate(
[
'uuid' => $item['uid'],
],
[
'name' => $item['name'],
'owner_name' => $item['owner_name'],
'application_type' => $item['application_type'],
'application_type_name' => $item['application_type_name'],
'condition' => $item['condition'],
'registration_number' => $item['registration_number'],
'document_number' => $item['document_number'],
'address' => $item['address'],
'status' => $item['status'],
'status_name' => $item['status_name'],
'slf_status' => $item['slf_status'] ?? null,
'slf_status_name' => $item['slf_status_name'] ?? null,
'function_type' => $item['function_type'],
'consultation_type' => $item['consultation_type'],
'due_date' => $item['due_date'],
'land_certificate_phase' => $item['land_certificate_phase'],
'task_created_at' => $taskCreatedAt,
]
);
Log::info("executed page", ['page' => $currentPage, 'total' => $totalPage]);
$tasksCollective = [];
foreach ($tasks as $item) {
try {
$tasksCollective[] = [
'uuid' => $item['uid'],
'name' => $item['name'],
'owner_name' => $item['owner_name'],
'application_type' => $item['application_type'],
'application_type_name' => $item['application_type_name'],
'condition' => $item['condition'],
'registration_number' => $item['registration_number'],
'document_number' => $item['document_number'],
'address' => $item['address'],
'status' => $item['status'],
'status_name' => $item['status_name'],
'slf_status' => $item['slf_status'] ?? null,
'slf_status_name' => $item['slf_status_name'] ?? null,
'function_type' => $item['function_type'],
'consultation_type' => $item['consultation_type'],
'due_date' => $item['due_date'],
'land_certificate_phase' => $item['land_certificate_phase'],
'task_created_at' => isset($item['created_at']) ? Carbon::parse($item['created_at'])->format('Y-m-d H:i:s') : null,
'updated_at' => now(),
'created_at' => now(),
];
// Synchronize additional details
$this->syncIndexIntegration($item['uid']);
$this->syncTaskDetailSubmit($item['uid']);
Log::info("executed page: ". $currentPage);
$savedCount++;
} catch (Exception $e) {
$importDatasource->update([
'status' => ImportDatasourceStatus::Failed->value,
'message' => 'failed to save',
'response_body' => $item
]);
$failedCount++;
Log::error("Failed to process task", [
'error' => $e->getMessage(),
'task' => $item,
]);
$failedCount++;
}
}
PbgTask::upsert($tasksCollective, ['uuid'], [
'name', 'owner_name', 'application_type', 'application_type_name', 'condition',
'registration_number', 'document_number', 'address', 'status', 'status_name',
'slf_status', 'slf_status_name', 'function_type', 'consultation_type', 'due_date',
'land_certificate_phase', 'task_created_at', 'updated_at'
]);
}
$result = [
"savedCount" => $savedCount,
"failedCount" => $failedCount,
];
$importDatasource->update([
'status' => ImportDatasourceStatus::Success->value,
'message' => "Successfully success data: " .$savedCount. " failed data : " .$failedCount
'message' => "Successfully processed: $savedCount, Failed: $failedCount"
]);
Log::info("syncTaskList completed", $result);
return $this->resSuccess($result);
Log::info("syncTaskList completed", ['savedCount' => $savedCount, 'failedCount' => $failedCount]);
return $this->resSuccess(['savedCount' => $savedCount, 'failedCount' => $failedCount]);
}
@@ -260,11 +233,11 @@ class ServiceSIMBG
}
$detailCreatedAt = isset($data['created_at'])
? \Carbon\Carbon::parse($data['created_at'])->format('Y-m-d H:i:s')
? Carbon::parse($data['created_at'])->format('Y-m-d H:i:s')
: null;
$detailUpdatedAt = isset($data['updated_at'])
? \Carbon\Carbon::parse($data['updated_at'])->format('Y-m-d H:i:s')
? Carbon::parse($data['updated_at'])->format('Y-m-d H:i:s')
: null;
PbgTaskRetributions::updateOrCreate(
@@ -292,24 +265,20 @@ class ServiceSIMBG
);
$prasaranaData = $data['prasarana'] ?? [];
if (is_array($prasaranaData) && count($prasaranaData) > 0) {
foreach ($prasaranaData as $item) {
PbgTaskPrasarana::updateOrCreate(
[
'pbg_task_uid' => $uuid,
'prasarana_id' => $item['id'] ?? null,
],
[
'pbg_task_uid' => $uuid,
'prasarana_type' => $item['prasarana_type'] ?? null,
'building_type' => $item['building_type'] ?? null,
'total' => $item['total'] ?? null,
'quantity' => $item['quantity'] ?? null,
'unit' => $item['unit'] ?? null,
'index_prasarana' => $item['index_prasarana'] ?? null,
]
);
}
if (!empty($prasaranaData)) {
$insertData = array_map(fn($item) => [
'pbg_task_uid' => $uuid,
'prasarana_id' => $item['id'] ?? null,
'prasarana_type' => $item['prasarana_type'] ?? null,
'building_type' => $item['building_type'] ?? null,
'total' => $item['total'] ?? null,
'quantity' => $item['quantity'] ?? null,
'unit' => $item['unit'] ?? null,
'index_prasarana' => $item['index_prasarana'] ?? null,
], $prasaranaData);
// Use bulk insert or upsert for faster database operation
PbgTaskPrasarana::upsert($insertData, ['pbg_task_uid', 'prasarana_id']);
}
Log::info("syncTaskDetailSubmit completed successfully", ['uuid' => $uuid]);