diff --git a/app/Console/Commands/InitSpatialPlanningDatas.php b/app/Console/Commands/InitSpatialPlanningDatas.php deleted file mode 100644 index 1cf9a2b..0000000 --- a/app/Console/Commands/InitSpatialPlanningDatas.php +++ /dev/null @@ -1,319 +0,0 @@ -argument('file') ?? 'public/templates/2025.xlsx'; - $fullPath = storage_path('app/' . $filePath); - - // Check if file exists - if (!file_exists($fullPath)) { - $this->error("File not found: {$fullPath}"); - $this->info("Available files in templates:"); - $this->listAvailableFiles(); - return 1; - } - - // Handle truncate options - if (($this->option('truncate') && $this->option('safe-truncate')) || - ($this->option('truncate') && $this->option('force-truncate')) || - ($this->option('safe-truncate') && $this->option('force-truncate'))) { - $this->error('Cannot use multiple truncate options together. Choose only one.'); - return 1; - } - - // Confirm truncate if requested - if ($this->option('truncate')) { - if ($this->confirm('This will delete all existing spatial planning data and related retribution proposals. Continue?')) { - $this->info('Truncating tables...'); - - try { - // First delete retribution proposals that reference spatial plannings - $deletedProposals = DB::table('retribution_proposals') - ->whereNotNull('spatial_planning_id') - ->count(); - - if ($deletedProposals > 0) { - DB::table('retribution_proposals') - ->whereNotNull('spatial_planning_id') - ->delete(); - $this->info("Deleted {$deletedProposals} retribution proposals linked to spatial plannings."); - } - - // Method 1: Try truncate with disabled foreign key checks - try { - DB::statement('SET FOREIGN_KEY_CHECKS = 0'); - DB::table('spatial_plannings')->truncate(); - DB::statement('SET FOREIGN_KEY_CHECKS = 1'); - $this->info('Spatial plannings table truncated successfully.'); - } catch (\Exception $truncateError) { - // Method 2: Fallback to delete if truncate fails - $this->warn('Truncate failed, using delete method...'); - $deletedSpatial = DB::table('spatial_plannings')->delete(); - $this->info("Deleted {$deletedSpatial} spatial planning records."); - - // Reset auto increment - DB::statement('ALTER TABLE spatial_plannings AUTO_INCREMENT = 1'); - } - - } catch (\Exception $e) { - $this->error('Failed to truncate tables: ' . $e->getMessage()); - return 1; - } - } else { - $this->info('Operation cancelled.'); - return 0; - } - } - - // Force truncate - disable foreign key checks and truncate everything - if ($this->option('force-truncate')) { - if ($this->confirm('This will FORCE truncate ALL spatial planning data by disabling foreign key checks. This is risky! Continue?')) { - $this->info('Force truncating with disabled foreign key checks...'); - - try { - DB::beginTransaction(); - - // Disable foreign key checks - DB::statement('SET FOREIGN_KEY_CHECKS = 0'); - - // Truncate both tables - DB::table('retribution_proposals')->truncate(); - DB::table('spatial_plannings')->truncate(); - - // Re-enable foreign key checks - DB::statement('SET FOREIGN_KEY_CHECKS = 1'); - - $this->info('Force truncate completed successfully.'); - - DB::commit(); - - } catch (\Exception $e) { - DB::rollBack(); - // Make sure to re-enable foreign key checks even on error - try { - DB::statement('SET FOREIGN_KEY_CHECKS = 1'); - } catch (\Exception $fkError) { - $this->error('Failed to re-enable foreign key checks: ' . $fkError->getMessage()); - } - $this->error('Failed to force truncate: ' . $e->getMessage()); - return 1; - } - } else { - $this->info('Operation cancelled.'); - return 0; - } - } - - // Safe truncate - only delete spatial plannings without retribution proposals - if ($this->option('safe-truncate')) { - if ($this->confirm('This will delete only spatial planning data that have no retribution proposals. Continue?')) { - $this->info('Safe truncating spatial plannings...'); - - try { - DB::beginTransaction(); - - // Count spatial plannings with retribution proposals - $withProposals = DB::table('spatial_plannings') - ->whereExists(function ($query) { - $query->select(DB::raw(1)) - ->from('retribution_proposals') - ->whereColumn('retribution_proposals.spatial_planning_id', 'spatial_plannings.id'); - }) - ->count(); - - // Delete spatial plannings without retribution proposals - $deletedCount = DB::table('spatial_plannings') - ->whereNotExists(function ($query) { - $query->select(DB::raw(1)) - ->from('retribution_proposals') - ->whereColumn('retribution_proposals.spatial_planning_id', 'spatial_plannings.id'); - }) - ->delete(); - - $this->info("Deleted {$deletedCount} spatial plannings without retribution proposals."); - $this->info("Kept {$withProposals} spatial plannings that have retribution proposals."); - - DB::commit(); - - } catch (\Exception $e) { - DB::rollBack(); - $this->error('Failed to safe truncate: ' . $e->getMessage()); - return 1; - } - } else { - $this->info('Operation cancelled.'); - return 0; - } - } - - $this->info("Starting import from: {$filePath}"); - $this->info("Full path: {$fullPath}"); - - try { - DB::beginTransaction(); - - $data = Excel::toArray([], $fullPath); - - if (empty($data) || empty($data[0])) { - $this->error('No data found in the file.'); - return 1; - } - - $rows = $data[0]; // Get first sheet - $headers = array_shift($rows); // Remove header row - - $this->info("Found " . count($rows) . " data rows to import."); - $this->info("Headers: " . implode(', ', $headers)); - - dd($rows[0]); - - $progressBar = $this->output->createProgressBar(count($rows)); - $progressBar->start(); - - $imported = 0; - $skipped = 0; - - foreach ($rows as $index => $row) { - try { - // Skip empty rows - if (empty(array_filter($row))) { - $skipped++; - $progressBar->advance(); - continue; - } - - // Map CSV columns to model attributes - $spatialData = [ - 'name' => $this->cleanString($row[1] ?? ''), // pemohon - 'location' => $this->cleanString($row[2] ?? ''), // alamat - 'activities' => $this->cleanString($row[3] ?? ''), // activities - 'land_area' => $this->cleanNumber($row[4] ?? 0), // luas_lahan - 'site_bcr' => $this->cleanNumber($row[5] ?? 0), // bcr_kawasan - 'area' => $this->cleanNumber($row[6] ?? 0), // area - 'no_tapak' => $this->cleanString($row[7] ?? ''), // no_tapak - 'no_skkl' => $this->cleanString($row[8] ?? ''), // no_skkl - 'no_ukl' => $this->cleanString($row[9] ?? ''), // no_ukl - 'building_function' => $this->cleanString($row[10] ?? ''), // fungsi_bangunan - 'sub_building_function' => $this->cleanString($row[11] ?? ''), // sub_fungsi_bangunan - 'number_of_floors' => $this->cleanNumber($row[12] ?? 1), // jumlah_lantai - 'number' => $this->cleanString($row[0] ?? ''), // no - 'date' => now(), // Set current date - 'kbli' => null, // Not in CSV, set as null - ]; - - // Validate required fields - if (empty($spatialData['name']) && empty($spatialData['activities'])) { - $skipped++; - $progressBar->advance(); - continue; - } - - SpatialPlanning::create($spatialData); - $imported++; - - } catch (Exception $e) { - $this->newLine(); - $this->error("Error importing row " . ($index + 2) . ": " . $e->getMessage()); - $skipped++; - } - - $progressBar->advance(); - } - - $progressBar->finish(); - $this->newLine(2); - - DB::commit(); - - $this->info("Import completed successfully!"); - $this->info("Imported: {$imported} records"); - $this->info("Skipped: {$skipped} records"); - - return 0; - - } catch (Exception $e) { - DB::rollBack(); - $this->error("Import failed: " . $e->getMessage()); - return 1; - } - } - - /** - * Clean string data - */ - private function cleanString($value) - { - if (is_null($value)) return null; - return trim(str_replace(["\n", "\r", "\t"], ' ', $value)); - } - - /** - * Clean numeric data - */ - private function cleanNumber($value) - { - if (is_null($value) || $value === '') return 0; - - // Remove non-numeric characters except decimal point - $cleaned = preg_replace('/[^0-9.]/', '', $value); - - return is_numeric($cleaned) ? (float) $cleaned : 0; - } - - /** - * List available template files - */ - private function listAvailableFiles() - { - $templatesPath = storage_path('app/public/templates'); - if (is_dir($templatesPath)) { - $this->info("Files in storage/app/public/templates:"); - $extensions = ['csv', 'xlsx', 'xls']; - foreach ($extensions as $ext) { - $files = glob($templatesPath . '/*.' . $ext); - foreach ($files as $file) { - $this->line(' - ' . basename($file)); - } - } - } - - $publicTemplatesPath = public_path('templates'); - if (is_dir($publicTemplatesPath)) { - $this->info("Files in public/templates:"); - $extensions = ['csv', 'xlsx', 'xls']; - foreach ($extensions as $ext) { - $files = glob($publicTemplatesPath . '/*.' . $ext); - foreach ($files as $file) { - $this->line(' - ' . basename($file)); - } - } - } - } -} diff --git a/app/Console/Commands/ScrapingData.php b/app/Console/Commands/ScrapingData.php deleted file mode 100644 index 6aa7d96..0000000 --- a/app/Console/Commands/ScrapingData.php +++ /dev/null @@ -1,45 +0,0 @@ -info("Scraping job dispatched successfully"); - } -} diff --git a/app/Console/Commands/StartScrapingData.php b/app/Console/Commands/StartScrapingData.php new file mode 100644 index 0000000..e036407 --- /dev/null +++ b/app/Console/Commands/StartScrapingData.php @@ -0,0 +1,80 @@ + PBG Task -> Details)'; + + /** + * Execute the console command. + */ + public function handle() + { + $this->info('🚀 Starting Optimized Scraping Data Job'); + $this->info('====================================='); + + if (!$this->option('confirm')) { + $this->warn('⚠️ This will start a comprehensive data scraping process:'); + $this->line(' 1. Google Sheet data scraping'); + $this->line(' 2. PBG Task parent data scraping'); + $this->line(' 3. Detailed task information scraping'); + $this->line(' 4. BigData resume generation'); + $this->newLine(); + + if (!$this->confirm('Do you want to continue?')) { + $this->info('Operation cancelled.'); + return 0; + } + } + + try { + // Dispatch the optimized job + $job = new ScrapingDataJob(); + dispatch($job); + + Log::info('ScrapingDataJob dispatched via command', [ + 'command' => $this->signature, + 'user' => $this->option('confirm') ? 'auto' : 'manual' + ]); + + $this->info('✅ Scraping Data Job has been dispatched to the scraping queue!'); + $this->newLine(); + $this->info('📊 Monitor the job with:'); + $this->line(' php artisan queue:monitor scraping'); + $this->newLine(); + $this->info('📜 View detailed logs with:'); + $this->line(' tail -f /var/log/supervisor/sibedas-queue-scraping.log | grep "SCRAPING DATA JOB"'); + $this->newLine(); + $this->info('🔍 Check ImportDatasource status:'); + $this->line(' docker-compose -f docker-compose.local.yml exec app php artisan tinker --execute="App\\Models\\ImportDatasource::latest()->first();"'); + + } catch (\Exception $e) { + $this->error('❌ Failed to dispatch ScrapingDataJob: ' . $e->getMessage()); + Log::error('Failed to dispatch ScrapingDataJob via command', [ + 'error' => $e->getMessage(), + 'trace' => $e->getTraceAsString() + ]); + return 1; + } + + return 0; + } +} diff --git a/app/Http/Controllers/RequestAssignment/PbgTaskController.php b/app/Http/Controllers/RequestAssignment/PbgTaskController.php index d772072..7716164 100644 --- a/app/Http/Controllers/RequestAssignment/PbgTaskController.php +++ b/app/Http/Controllers/RequestAssignment/PbgTaskController.php @@ -56,10 +56,37 @@ class PbgTaskController extends Controller */ public function show(string $id) { - $data = PbgTask::with(['pbg_task_retributions','pbg_task_index_integrations', 'pbg_task_retributions.pbg_task_prasarana', 'pbg_task_detail'])->findOrFail($id); + $data = PbgTask::with([ + 'pbg_task_retributions', + 'pbg_task_index_integrations', + 'pbg_task_retributions.pbg_task_prasarana', + 'pbg_task_detail', + 'dataLists' => function($query) { + $query->orderBy('data_type')->orderBy('name'); + } + ])->findOrFail($id); + + // Group data lists by data_type for easier display + $dataListsByType = $data->dataLists->groupBy('data_type'); + + // Debug: Log the data types found for this task + \Log::info('PBG Task Data Lists', [ + 'task_uuid' => $data->uuid, + 'total_data_lists' => $data->dataLists->count(), + 'data_types_found' => $dataListsByType->keys()->toArray(), + 'data_types_with_names' => $dataListsByType->map(function($items, $type) { + return [ + 'type' => $type, + 'name' => $items->first()->data_type_name ?? "Type {$type}", + 'count' => $items->count() + ]; + })->values()->toArray() + ]); + $statusOptions = PbgTaskStatus::getStatuses(); $applicationTypes = PbgTaskApplicationTypes::labels(); - return view("pbg_task.show", compact("data", 'statusOptions', 'applicationTypes')); + + return view("pbg_task.show", compact("data", 'statusOptions', 'applicationTypes', 'dataListsByType')); } /** diff --git a/app/Jobs/RetrySyncronizeJob.php b/app/Jobs/RetrySyncronizeJob.php index c9c5352..9b7eb59 100644 --- a/app/Jobs/RetrySyncronizeJob.php +++ b/app/Jobs/RetrySyncronizeJob.php @@ -48,8 +48,7 @@ class RetrySyncronizeJob implements ShouldQueue $data_setting_result = $service_google_sheet->get_big_resume_data(); - BigdataResume::generateResumeData($failed_import->id, "all", $data_setting_result); - BigdataResume::generateResumeData($failed_import->id, now()->year, $data_setting_result); + BigdataResume::generateResumeData($failed_import->id, "simbg", $data_setting_result); $failed_import->update([ 'status' => ImportDatasourceStatus::Success->value, diff --git a/app/Jobs/ScrapingDataJob.php b/app/Jobs/ScrapingDataJob.php index b348a82..633b65e 100644 --- a/app/Jobs/ScrapingDataJob.php +++ b/app/Jobs/ScrapingDataJob.php @@ -4,6 +4,7 @@ namespace App\Jobs; use App\Models\BigdataResume; use App\Models\ImportDatasource; +use App\Models\PbgTask; use App\Services\ServiceGoogleSheet; use App\Services\ServicePbgTask; use App\Services\ServiceTabPbgTask; @@ -21,72 +22,208 @@ class ScrapingDataJob implements ShouldQueue use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; /** - * Inject dependencies instead of creating them inside. + * Create a new job instance. */ - public function __construct( - ) { + public function __construct() + { + // Use dedicated scraping queue + $this->queue = 'scraping'; } /** - * Execute the job. + * Execute the job with optimized schema: + * 1. Scrape Google Sheet first + * 2. Scrape PBG Task to get parent data + * 3. Loop through parent tasks to scrape details via ServiceTabPbgTask */ public function handle() { + $import_datasource = null; $failed_uuid = null; - try { + $processedTasks = 0; + $totalTasks = 0; - $client = app(Client::class); + try { + Log::info("=== SCRAPING DATA JOB STARTED ==="); + + // Initialize services + $service_google_sheet = app(ServiceGoogleSheet::class); $service_pbg_task = app(ServicePbgTask::class); $service_tab_pbg_task = app(ServiceTabPbgTask::class); - $service_google_sheet = app(ServiceGoogleSheet::class); - $service_token = app(ServiceTokenSIMBG::class); - // Create a record with "processing" status + + // Create ImportDatasource record $import_datasource = ImportDatasource::create([ - 'message' => 'Initiating scraping...', + 'message' => 'Starting optimized scraping process...', 'response_body' => null, 'status' => 'processing', 'start_time' => now(), 'failed_uuid' => null ]); - // Run the scraping services + Log::info("ImportDatasource created", ['id' => $import_datasource->id]); + + // STEP 1: Scrape Google Sheet data first + Log::info("=== STEP 1: SCRAPING GOOGLE SHEET ==="); + $import_datasource->update(['message' => 'Scraping Google Sheet data...']); + $service_google_sheet->run_service(); + Log::info("Google Sheet scraping completed successfully"); + + // STEP 2: Scrape PBG Task to get parent data + Log::info("=== STEP 2: SCRAPING PBG TASK PARENT DATA ==="); + $import_datasource->update(['message' => 'Scraping PBG Task parent data...']); + $service_pbg_task->run_service(); - try{ - $service_tab_pbg_task->run_service(); - }catch(\Exception $e){ - $failed_uuid = $service_tab_pbg_task->getFailedUUID(); - throw $e; - } + Log::info("PBG Task parent data scraping completed"); + // STEP 3: Get all PBG tasks for detail scraping + $totalTasks = PbgTask::count(); + Log::info("=== STEP 3: SCRAPING PBG TASK DETAILS ===", [ + 'total_tasks' => $totalTasks + ]); + + $import_datasource->update([ + 'message' => "Scraping details for {$totalTasks} PBG tasks..." + ]); + + // Process tasks in chunks for memory efficiency + $chunkSize = 100; + $processedTasks = 0; + + PbgTask::orderBy('id')->chunk($chunkSize, function ($pbg_tasks) use ( + $service_tab_pbg_task, + &$processedTasks, + $totalTasks, + $import_datasource, + &$failed_uuid + ) { + foreach ($pbg_tasks as $pbg_task) { + try { + // Scrape all details for this task + $this->processTaskDetails($service_tab_pbg_task, $pbg_task->uuid); + + $processedTasks++; + + // Update progress every 10 tasks + if ($processedTasks % 10 === 0) { + $progress = round(($processedTasks / $totalTasks) * 100, 2); + Log::info("Progress update", [ + 'processed' => $processedTasks, + 'total' => $totalTasks, + 'progress' => "{$progress}%" + ]); + + $import_datasource->update([ + 'message' => "Processing details: {$processedTasks}/{$totalTasks} ({$progress}%)" + ]); + } + + } catch (\Exception $e) { + Log::warning("Failed to process task details", [ + 'uuid' => $pbg_task->uuid, + 'error' => $e->getMessage() + ]); + + // Store failed UUID but continue processing + $failed_uuid = $pbg_task->uuid; + + // Only stop if it's a critical error + if ($this->isCriticalError($e)) { + throw $e; + } + } + } + }); + + Log::info("Task details scraping completed", [ + 'processed_tasks' => $processedTasks, + 'total_tasks' => $totalTasks + ]); + + // STEP 4: Generate BigData Resume + Log::info("=== STEP 4: GENERATING BIGDATA RESUME ==="); + $import_datasource->update(['message' => 'Generating BigData resume...']); + $data_setting_result = $service_google_sheet->get_big_resume_data(); + BigdataResume::generateResumeData($import_datasource->id, "simbg", $data_setting_result); + + Log::info("BigData resume generated successfully"); - // BigdataResume::generateResumeData($import_datasource->id, "all", $data_setting_result); - BigdataResume::generateResumeData($import_datasource->id, now()->year, $data_setting_result); - - // Update status to success + // Update final status $import_datasource->update([ 'status' => 'success', - 'message' => 'Scraping completed successfully.', - 'finish_time' => now() + 'message' => "Scraping completed successfully. Processed {$processedTasks}/{$totalTasks} tasks.", + 'finish_time' => now(), + 'failed_uuid' => $failed_uuid // Store last failed UUID if any + ]); + + Log::info("=== SCRAPING DATA JOB COMPLETED SUCCESSFULLY ===", [ + 'import_datasource_id' => $import_datasource->id, + 'processed_tasks' => $processedTasks, + 'total_tasks' => $totalTasks, + 'has_failures' => !is_null($failed_uuid) ]); } catch (\Exception $e) { - Log::error('Scraping failed: ' . $e->getMessage(), ['trace' => $e->getTraceAsString()]); + Log::error('=== SCRAPING DATA JOB FAILED ===', [ + 'error' => $e->getMessage(), + 'file' => $e->getFile(), + 'line' => $e->getLine(), + 'processed_tasks' => $processedTasks, + 'total_tasks' => $totalTasks, + 'failed_uuid' => $failed_uuid, + 'trace' => $e->getTraceAsString() + ]); - // Update status to failed - if (isset($import_datasource)) { + // Update ImportDatasource with failure info + if ($import_datasource) { $import_datasource->update([ 'status' => 'failed', - 'message' => 'Terjadi kesalahan, Syncronize tidak selesai', - 'response_body' => 'Terjadi kesalahan, Syncronize tidak selesai', + 'message' => "Scraping failed: {$e->getMessage()}. Processed {$processedTasks}/{$totalTasks} tasks.", + 'response_body' => 'Scraping process interrupted due to error', 'finish_time' => now(), 'failed_uuid' => $failed_uuid, ]); } - // Mark the job as failed + // Don't retry this job $this->fail($e); } } + + /** + * Process all detail endpoints for a single PBG task + */ + private function processTaskDetails(ServiceTabPbgTask $service, string $uuid): void + { + // Call all detail scraping methods for this task + $service->scraping_task_details($uuid); + $service->scraping_pbg_data_list($uuid); + $service->scraping_task_retributions($uuid); + $service->scraping_task_integrations($uuid); + } + + /** + * Determine if an error is critical enough to stop the entire process + */ + private function isCriticalError(\Exception $e): bool + { + $criticalMessages = [ + 'authentication failed', + 'token expired', + 'database connection', + 'memory exhausted', + 'maximum execution time' + ]; + + $errorMessage = strtolower($e->getMessage()); + + foreach ($criticalMessages as $critical) { + if (strpos($errorMessage, $critical) !== false) { + return true; + } + } + + return false; + } } diff --git a/app/Models/PbgTask.php b/app/Models/PbgTask.php index 2986747..59c8900 100644 --- a/app/Models/PbgTask.php +++ b/app/Models/PbgTask.php @@ -54,4 +54,86 @@ class PbgTask extends Model public function attachments(){ return $this->hasMany(PbgTaskAttachment::class, 'pbg_task_id', 'id'); } + + /** + * Get the data lists associated with this PBG task (One to Many) + * One pbg_task can have many data lists + */ + public function dataLists() + { + return $this->hasMany(PbgTaskDetailDataList::class, 'pbg_task_uuid', 'uuid'); + } + + /** + * Get only data lists with files + */ + public function dataListsWithFiles() + { + return $this->hasMany(PbgTaskDetailDataList::class, 'pbg_task_uuid', 'uuid') + ->whereNotNull('file') + ->where('file', '!=', ''); + } + + /** + * Get data lists by status + */ + public function dataListsByStatus($status) + { + return $this->hasMany(PbgTaskDetailDataList::class, 'pbg_task_uuid', 'uuid') + ->where('status', $status); + } + + /** + * Get data lists by data type + */ + public function dataListsByType($dataType) + { + return $this->hasMany(PbgTaskDetailDataList::class, 'pbg_task_uuid', 'uuid') + ->where('data_type', $dataType); + } + + /** + * Create or update data lists from API response + */ + public function syncDataLists(array $dataLists): void + { + foreach ($dataLists as $listData) { + PbgTaskDetailDataList::updateOrCreate( + ['uid' => $listData['uid']], + [ + 'name' => $listData['name'] ?? null, + 'description' => $listData['description'] ?? null, + 'status' => $listData['status'] ?? null, + 'status_name' => $listData['status_name'] ?? null, + 'data_type' => $listData['data_type'] ?? null, + 'data_type_name' => $listData['data_type_name'] ?? null, + 'file' => $listData['file'] ?? null, + 'note' => $listData['note'] ?? null, + 'pbg_task_uuid' => $this->uuid, + ] + ); + } + } + + /** + * Get data lists count by status + */ + public function getDataListsCountByStatusAttribute() + { + return $this->dataLists() + ->selectRaw('status, COUNT(*) as count') + ->groupBy('status') + ->pluck('count', 'status'); + } + + /** + * Get data lists count by data type + */ + public function getDataListsCountByTypeAttribute() + { + return $this->dataLists() + ->selectRaw('data_type, COUNT(*) as count') + ->groupBy('data_type') + ->pluck('count', 'data_type'); + } } diff --git a/app/Models/PbgTaskDetail.php b/app/Models/PbgTaskDetail.php index a0e1526..55dfa5f 100644 --- a/app/Models/PbgTaskDetail.php +++ b/app/Models/PbgTaskDetail.php @@ -141,6 +141,8 @@ class PbgTaskDetail extends Model return $this->belongsTo(PbgTask::class, 'pbg_task_uid', 'uuid'); } + + /** * Create or update PbgTaskDetail from API response */ @@ -249,4 +251,6 @@ class PbgTaskDetail extends Model $detailData ); } + + } diff --git a/app/Models/PbgTaskDetailDataList.php b/app/Models/PbgTaskDetailDataList.php new file mode 100644 index 0000000..283256a --- /dev/null +++ b/app/Models/PbgTaskDetailDataList.php @@ -0,0 +1,161 @@ + 'integer', + 'data_type' => 'integer', + 'created_at' => 'datetime', + 'updated_at' => 'datetime', + ]; + + /** + * Relationship to PbgTask (Many to One) + * Many data lists belong to one pbg_task + */ + public function pbgTask() + { + return $this->belongsTo(PbgTask::class, 'pbg_task_uuid', 'uuid'); + } + + /** + * Get the full file path + */ + public function getFilePathAttribute() + { + return $this->file ? storage_path('app/public/' . $this->file) : null; + } + + /** + * Get the file URL + */ + public function getFileUrlAttribute() + { + return $this->file ? asset('storage/' . $this->file) : null; + } + + /** + * Check if file exists + */ + public function hasFile() + { + return !empty($this->file) && file_exists($this->getFilePathAttribute()); + } + + /** + * Get status badge color based on status + */ + public function getStatusBadgeAttribute() + { + return match($this->status) { + 1 => 'success', // Sesuai + 0 => 'danger', // Tidak Sesuai + default => 'secondary' + }; + } + + /** + * Scope: Filter by status + */ + public function scopeByStatus($query, $status) + { + return $query->where('status', $status); + } + + /** + * Scope: Filter by data type + */ + public function scopeByDataType($query, $dataType) + { + return $query->where('data_type', $dataType); + } + + /** + * Scope: With files only + */ + public function scopeWithFiles($query) + { + return $query->whereNotNull('file')->where('file', '!=', ''); + } + + /** + * Scope: Search by name or description + */ + public function scopeSearch($query, $search) + { + return $query->where(function ($q) use ($search) { + $q->where('name', 'LIKE', "%{$search}%") + ->orWhere('description', 'LIKE', "%{$search}%") + ->orWhere('status_name', 'LIKE', "%{$search}%") + ->orWhere('data_type_name', 'LIKE', "%{$search}%"); + }); + } + + /** + * Get file extension from file path + */ + public function getFileExtensionAttribute() + { + if (!$this->file) { + return null; + } + return strtoupper(pathinfo($this->file, PATHINFO_EXTENSION)); + } + + /** + * Get filename from file path + */ + public function getFileNameAttribute() + { + if (!$this->file) { + return null; + } + return basename($this->file); + } + + /** + * Get formatted created date + */ + public function getFormattedCreatedAtAttribute() + { + return $this->created_at ? $this->created_at->format('d M Y, H:i') : '-'; + } + + /** + * Get truncated description + */ + public function getTruncatedDescriptionAttribute() + { + return $this->description ? \Str::limit($this->description, 80) : null; + } + + /** + * Get truncated note + */ + public function getTruncatedNoteAttribute() + { + return $this->note ? \Str::limit($this->note, 100) : null; + } +} \ No newline at end of file diff --git a/app/Services/ServicePbgTask.php b/app/Services/ServicePbgTask.php index 2a221fa..73fc234 100644 --- a/app/Services/ServicePbgTask.php +++ b/app/Services/ServicePbgTask.php @@ -106,7 +106,7 @@ class ServicePbgTask }; do { - $url = "{$this->simbg_host}/api/pbg/v1/list/?page={$currentPage}&size={$this->fetch_per_page}&sort=ASC&date&search&status&slf_status&type&sort_by=created_at&application_type=1&start_date&end_date"; + $url = "{$this->simbg_host}/api/pbg/v1/list/?page={$currentPage}&size={$this->fetch_per_page}&sort=ASC&date&search&status&slf_status&type=task&sort_by=created_at&application_type=1&start_date&end_date"; $fetch_data = $fetchData($url); if (!$fetch_data) { diff --git a/app/Services/ServiceTabPbgTask.php b/app/Services/ServiceTabPbgTask.php index aefcb58..6613b73 100644 --- a/app/Services/ServiceTabPbgTask.php +++ b/app/Services/ServiceTabPbgTask.php @@ -5,6 +5,7 @@ namespace App\Services; use App\Models\GlobalSetting; use App\Models\PbgTask; use App\Models\PbgTaskDetail; +use App\Models\PbgTaskDetailDataList; use App\Models\PbgTaskIndexIntegrations; use App\Models\PbgTaskPrasarana; use App\Models\PbgTaskRetributions; @@ -36,44 +37,119 @@ class ServiceTabPbgTask $this->user_refresh_token = $auth_data['refresh']; } - public function run_service($retry_uuid = null) + public function run_service($retry_uuid = null, $chunk_size = 50) { try { - $pbg_tasks = PbgTask::orderBy('id')->get(); - $start = false; + $query = PbgTask::orderBy('id'); + + // If retry_uuid is provided, start from that UUID + if ($retry_uuid) { + $retryTask = PbgTask::where('uuid', $retry_uuid)->first(); + if ($retryTask) { + $query->where('id', '>=', $retryTask->id); + Log::info("Resuming sync from UUID: {$retry_uuid} (ID: {$retryTask->id})"); + } + } - foreach ($pbg_tasks as $pbg_task) { - if($retry_uuid){ - if($pbg_task->uuid === $retry_uuid){ - $start = true; - } + $totalTasks = $query->count(); + $processedCount = 0; + + Log::info("Starting sync for {$totalTasks} PBG Tasks with chunk size: {$chunk_size}"); - if(!$start){ + // Process in chunks to reduce memory usage + $query->chunk($chunk_size, function ($pbg_tasks) use (&$processedCount, $totalTasks) { + $chunkStartTime = now(); + + foreach ($pbg_tasks as $pbg_task) { + try { + $this->current_uuid = $pbg_task->uuid; + $taskStartTime = now(); + + // Process all endpoints for this task + $this->processTaskEndpoints($pbg_task->uuid); + + $processedCount++; + $taskTime = now()->diffInSeconds($taskStartTime); + + // Log progress every 10 tasks + if ($processedCount % 10 === 0) { + $progress = round(($processedCount / $totalTasks) * 100, 2); + Log::info("Progress: {$processedCount}/{$totalTasks} ({$progress}%) - Last task took {$taskTime}s"); + } + + } catch (\Exception $e) { + Log::error("Failed on UUID: {$this->current_uuid}, Error: " . $e->getMessage()); + + // Check if this is a critical error that should stop the process + if ($this->isCriticalError($e)) { + throw $e; + } + + // For non-critical errors, log and continue + Log::warning("Skipping UUID {$this->current_uuid} due to non-critical error"); continue; } } - try{ - $this->current_uuid = $pbg_task->uuid; - $this->scraping_task_details($pbg_task->uuid); - // $this->scraping_task_assignments($pbg_task->uuid); - $this->scraping_task_retributions($pbg_task->uuid); - $this->scraping_task_integrations($pbg_task->uuid); - }catch(\Exception $e){ - Log::error("Failed on UUID: {$this->current_uuid}, Error: " . $e->getMessage()); - throw $e; + + $chunkTime = now()->diffInSeconds($chunkStartTime); + Log::info("Processed chunk of {$pbg_tasks->count()} tasks in {$chunkTime} seconds"); + + // Small delay between chunks to prevent API rate limiting + if ($pbg_tasks->count() === $chunk_size) { + sleep(1); } - } + }); + + Log::info("Successfully completed sync for {$processedCount} PBG Tasks"); + } catch (\Exception $e) { - Log::error("Failed to syncronize: " . $e->getMessage()); + Log::error("Failed to synchronize: " . $e->getMessage()); throw $e; } } + /** + * Process all endpoints for a single task + */ + private function processTaskEndpoints(string $uuid): void + { + $this->scraping_task_details($uuid); + $this->scraping_pbg_data_list($uuid); + // $this->scraping_task_assignments($uuid); + $this->scraping_task_retributions($uuid); + $this->scraping_task_integrations($uuid); + } + + /** + * Determine if an error is critical and should stop the process + */ + private function isCriticalError(\Exception $e): bool + { + $message = $e->getMessage(); + + // Critical authentication errors + if (strpos($message, 'Token refresh and login failed') !== false) { + return true; + } + + // Critical system errors + if (strpos($message, 'Connection refused') !== false) { + return true; + } + + // Database connection errors + if (strpos($message, 'database') !== false && strpos($message, 'connection') !== false) { + return true; + } + + return false; + } + public function getFailedUUID(){ return $this->current_uuid; } - private function scraping_task_details($uuid) + public function scraping_task_details($uuid) { $url = "{$this->simbg_host}/api/pbg/v1/detail/{$uuid}/"; $options = [ @@ -144,7 +220,7 @@ class ServiceTabPbgTask throw new \Exception("Failed to fetch task details for UUID {$uuid} after retries."); } - private function scraping_task_assignments($uuid) + public function scraping_task_assignments($uuid) { $url = "{$this->simbg_host}/api/pbg/v1/list-tim-penilai/{$uuid}/?page=1&size=10"; $options = [ @@ -237,7 +313,173 @@ class ServiceTabPbgTask throw new \Exception("Failed to fetch task assignments for UUID {$uuid} after retries."); } - private function scraping_task_retributions($uuid) + public function scraping_pbg_data_list($uuid){ + $url = "{$this->simbg_host}/api/pbg/v1/detail/{$uuid}/list-data/?sort=DESC"; + $options = [ + 'headers' => [ + 'Authorization' => "Bearer {$this->user_token}", + 'Content-Type' => 'application/json' + ] + ]; + + $maxRetries = 3; + $initialDelay = 1; + $retriedAfter401 = false; + + for ($retryCount = 0; $retryCount < $maxRetries; $retryCount++) { + try{ + $response = $this->client->get($url, $options); + $responseData = json_decode($response->getBody()->getContents(), true, 512, JSON_THROW_ON_ERROR); + + if (empty($responseData['data']) || !is_array($responseData['data'])) { + Log::info("No data list found for UUID: {$uuid}"); + return true; + } + + $data = $responseData['data']; + + Log::info("Processing data list for UUID: {$uuid}, found " . count($data) . " items"); + + // Process each data list item and save to database + $this->processDataListItems($data, $uuid); + + return $responseData; + } catch (\GuzzleHttp\Exception\ClientException $e) { + if ($e->getCode() === 401 && !$retriedAfter401) { + Log::warning("401 Unauthorized - Refreshing token and retrying..."); + try{ + $this->refreshToken(); + $options['headers']['Authorization'] = "Bearer {$this->user_token}"; + $retriedAfter401 = true; + continue; + }catch(\Exception $refreshError){ + Log::error("Token refresh and login failed: " . $refreshError->getMessage()); + return false; + } + } + + return false; + } catch (\GuzzleHttp\Exception\ServerException | \GuzzleHttp\Exception\ConnectException $e) { + if ($e->getCode() === 502) { + Log::warning("502 Bad Gateway - Retrying in {$initialDelay} seconds..."); + } else { + Log::error("Network error ({$e->getCode()}) - Retrying in {$initialDelay} seconds..."); + } + + sleep($initialDelay); + $initialDelay *= 2; + } catch (\GuzzleHttp\Exception\RequestException $e) { + Log::error("Request error ({$e->getCode()}): " . $e->getMessage()); + return false; + } catch (\JsonException $e) { + Log::error("JSON decoding error: " . $e->getMessage()); + return false; + } catch (\Throwable $e) { + Log::critical("Unhandled error: " . $e->getMessage(), ['trace' => $e->getTraceAsString()]); + return false; + } + } + + Log::error("Failed to fetch task data list for UUID {$uuid} after {$maxRetries} retries."); + throw new \Exception("Failed to fetch task data list for UUID {$uuid} after retries."); + } + + /** + * Process and save data list items to database (Optimized with bulk operations) + */ + private function processDataListItems(array $dataListItems, string $pbgTaskUuid): void + { + try { + if (empty($dataListItems)) { + return; + } + + $batchData = []; + $validItems = 0; + + foreach ($dataListItems as $item) { + // Validate required fields + if (empty($item['uid'])) { + Log::warning("Skipping data list item with missing UID for PBG Task: {$pbgTaskUuid}"); + continue; + } + + // Parse created_at if exists + $createdAt = null; + if (!empty($item['created_at'])) { + try { + $createdAt = Carbon::parse($item['created_at'])->format('Y-m-d H:i:s'); + } catch (\Exception $e) { + Log::warning("Invalid created_at format for data list UID: {$item['uid']}, Error: " . $e->getMessage()); + } + } + + $batchData[] = [ + 'uid' => $item['uid'], + 'name' => $item['name'] ?? null, + 'description' => $item['description'] ?? null, + 'status' => $item['status'] ?? null, + 'status_name' => $item['status_name'] ?? null, + 'data_type' => $item['data_type'] ?? null, + 'data_type_name' => $item['data_type_name'] ?? null, + 'file' => $item['file'] ?? null, + 'note' => $item['note'] ?? null, + 'pbg_task_uuid' => $pbgTaskUuid, + 'created_at' => $createdAt ?: now(), + 'updated_at' => now(), + ]; + + $validItems++; + } + + if (!empty($batchData)) { + // Use upsert for bulk insert/update operations + PbgTaskDetailDataList::upsert( + $batchData, + ['uid'], // Unique columns + [ + 'name', 'description', 'status', 'status_name', + 'data_type', 'data_type_name', 'file', 'note', + 'pbg_task_uuid', 'updated_at' + ] // Columns to update + ); + + Log::info("Successfully bulk processed {$validItems} data list items for PBG Task: {$pbgTaskUuid}"); + } + + } catch (\Exception $e) { + Log::error("Error bulk processing data list items for PBG Task {$pbgTaskUuid}: " . $e->getMessage()); + throw $e; + } + } + + /** + * Alternative method using PbgTask model's syncDataLists for cleaner code + */ + private function processDataListItemsWithModel(array $dataListItems, string $pbgTaskUuid): void + { + try { + // Find the PbgTask + $pbgTask = PbgTask::where('uuid', $pbgTaskUuid)->first(); + + if (!$pbgTask) { + Log::error("PBG Task not found with UUID: {$pbgTaskUuid}"); + return; + } + + // Use the model's syncDataLists method + $pbgTask->syncDataLists($dataListItems); + + $processedCount = count($dataListItems); + Log::info("Successfully synced {$processedCount} data list items for PBG Task: {$pbgTaskUuid} using model method"); + + } catch (\Exception $e) { + Log::error("Error syncing data list items for PBG Task {$pbgTaskUuid}: " . $e->getMessage()); + throw $e; + } + } + + public function scraping_task_retributions($uuid) { $url = "{$this->simbg_host}/api/pbg/v1/detail/" . $uuid . "/retribution/submit/"; $options = [ @@ -354,7 +596,7 @@ class ServiceTabPbgTask throw new \Exception("Failed to fetch task retributions for UUID {$uuid} after retries."); } - private function scraping_task_integrations($uuid){ + public function scraping_task_integrations($uuid){ $url = "{$this->simbg_host}/api/pbg/v1/detail/" . $uuid . "/retribution/indeks-terintegrasi/"; $options = [ 'headers' => [ diff --git a/database/migrations/2025_08_15_133036_create_pbg_task_detail_data_lists_table.php b/database/migrations/2025_08_15_133036_create_pbg_task_detail_data_lists_table.php new file mode 100644 index 0000000..f96a497 --- /dev/null +++ b/database/migrations/2025_08_15_133036_create_pbg_task_detail_data_lists_table.php @@ -0,0 +1,47 @@ +id(); + $table->string('uid')->unique(); // UID from response + $table->string('name'); // Nama data + $table->text('description')->nullable(); // Deskripsi (bisa null) + $table->integer('status')->nullable(); // Status (1 = Sesuai, etc) + $table->string('status_name')->nullable(); // Nama status + $table->integer('data_type')->nullable(); // Tipe data (1 = Data Teknis Tanah, etc) + $table->string('data_type_name')->nullable(); // Nama tipe data + $table->text('file')->nullable(); // Path file + $table->text('note')->nullable(); // Catatan + + // Foreign key ke pbg_task (1 to many relationship) + $table->string('pbg_task_uuid')->nullable(); + $table->foreign('pbg_task_uuid')->references('uuid')->on('pbg_task')->onDelete('cascade'); + + // Indexes for better performance + $table->index('uid'); + $table->index('pbg_task_uuid'); + $table->index('status'); + $table->index('data_type'); + + $table->timestamps(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::dropIfExists('pbg_task_detail_data_lists'); + } +}; diff --git a/resources/scss/pages/pbg-task/show.scss b/resources/scss/pages/pbg-task/show.scss new file mode 100644 index 0000000..2f17e64 --- /dev/null +++ b/resources/scss/pages/pbg-task/show.scss @@ -0,0 +1,214 @@ +// PBG Task Show Page Styles +// Custom styles for data lists display (List Layout) + +.data-list-section { + .section-header { + border-bottom: 2px solid #f8f9fa; + padding-bottom: 0.75rem; + + .section-icon { + font-size: 1.2rem; + color: #5d87ff; + } + + .section-title { + font-size: 1.1rem; + font-weight: 600; + color: #2c3e50; + } + + .section-count { + font-size: 0.85rem; + } + } +} + +.data-list-container { + .data-list-item { + border: 1px solid #e9ecef; + border-radius: 6px; + margin-bottom: 0.75rem; + padding: 1rem; + background: #ffffff; + transition: all 0.2s ease; + position: relative; + + &:hover { + border-color: #5d87ff; + box-shadow: 0 2px 8px rgba(93, 135, 255, 0.1); + transform: translateX(2px); + } + + &::before { + content: ""; + position: absolute; + left: 0; + top: 0; + bottom: 0; + width: 3px; + background: #5d87ff; + border-radius: 3px 0 0 3px; + opacity: 0; + transition: opacity 0.2s ease; + } + + &:hover::before { + opacity: 1; + } + + .list-item-header { + .item-title { + font-size: 1rem; + font-weight: 600; + color: #2c3e50; + line-height: 1.4; + + .item-number { + color: #5d87ff; + font-weight: 700; + margin-right: 0.5rem; + } + } + + .item-description { + font-size: 0.9rem; + line-height: 1.4; + color: #6c757d; + margin-bottom: 0.5rem; + } + + .item-status { + .badge { + font-size: 0.75rem; + padding: 0.25rem 0.6rem; + } + } + } + + .list-item-meta { + margin-top: 0.75rem; + padding-top: 0.75rem; + border-top: 1px solid #f1f3f4; + + small { + font-size: 0.8rem; + + i { + opacity: 0.7; + } + + .text-dark { + font-weight: 500; + } + + .badge { + font-size: 0.7rem; + padding: 0.1rem 0.4rem; + } + } + } + } + + // Empty state for individual sections + .no-items { + text-align: center; + padding: 2rem; + color: #6c757d; + font-style: italic; + border: 2px dashed #dee2e6; + border-radius: 6px; + + i { + font-size: 2rem; + margin-bottom: 0.5rem; + opacity: 0.5; + } + } +} + +// Empty state styles +.empty-state { + text-align: center; + padding: 3rem 1rem; + + .empty-icon { + font-size: 3rem; + color: #6c757d; + margin-bottom: 1rem; + opacity: 0.6; + } + + .empty-title { + font-size: 1.25rem; + color: #6c757d; + margin-bottom: 0.5rem; + } + + .empty-text { + color: #6c757d; + opacity: 0.8; + } +} + +// Badge variations +.badge { + &.bg-success { + background-color: #28a745 !important; + } + + &.bg-danger { + background-color: #dc3545 !important; + } + + &.bg-info { + background-color: #17a2b8 !important; + } + + &.bg-light { + background-color: #f8f9fa !important; + border: 1px solid #dee2e6; + } +} + +// Tab content spacing +#pbgTaskDataLists { + .section-divider { + border-bottom: 1px solid #e9ecef; + margin-bottom: 2rem; + padding-bottom: 1rem; + + &:last-child { + border-bottom: none; + margin-bottom: 0; + } + } +} + +// Responsive adjustments +@media (max-width: 768px) { + .data-list-container { + .data-list-item { + padding: 0.75rem; + + .list-item-header { + .item-title { + font-size: 0.95rem; + } + } + + .list-item-meta { + small { + font-size: 0.75rem; + } + } + } + } + + .data-list-section { + .section-header { + .section-title { + font-size: 1rem; + } + } + } +} diff --git a/resources/views/layouts/partials/topbar.blade.php b/resources/views/layouts/partials/topbar.blade.php index 5f339a3..3001cd0 100644 --- a/resources/views/layouts/partials/topbar.blade.php +++ b/resources/views/layouts/partials/topbar.blade.php @@ -38,7 +38,7 @@ -
- + --}}+ {{ $dataList->description }} +
+ @endif +There are no data lists associated with this PBG task.
+