arifal/sibedas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add new url scraping data and create tab data lists

Browse Source
This commit is contained in:
arifal
2025-08-15 17:25:20 +07:00
parent 6896fd62a3
commit 209ef07f9c
18 changed files with 1220 additions and 424 deletions
Download Patch File Download Diff File Expand all files Collapse all files

319
app/Console/Commands/InitSpatialPlanningDatas.php
View File

@@ -1,319 +0,0 @@
<?php
namespace App\Console\Commands;
use App\Models\SpatialPlanning;
use Illuminate\Console\Command;
use Maatwebsite\Excel\Facades\Excel;
use Illuminate\Support\Facades\DB;
use Exception;
class InitSpatialPlanningDatas extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'spatial:init {file? : Path to the CSV/Excel file} {--truncate : Clear existing data before import} {--safe-truncate : Clear only spatial plannings without retribution proposals} {--force-truncate : Force truncate by disabling foreign key checks}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Import spatial planning data from CSV/Excel file for retribution';
/**
* Execute the console command.
*/
public function handle()
{
$filePath = $this->argument('file') ?? 'public/templates/2025.xlsx';
$fullPath = storage_path('app/' . $filePath);
// Check if file exists
if (!file_exists($fullPath)) {
$this->error("File not found: {$fullPath}");
$this->info("Available files in templates:");
$this->listAvailableFiles();
return 1;
}
// Handle truncate options
if (($this->option('truncate') && $this->option('safe-truncate')) ||
($this->option('truncate') && $this->option('force-truncate')) ||
($this->option('safe-truncate') && $this->option('force-truncate'))) {
$this->error('Cannot use multiple truncate options together. Choose only one.');
return 1;
}
// Confirm truncate if requested
if ($this->option('truncate')) {
if ($this->confirm('This will delete all existing spatial planning data and related retribution proposals. Continue?')) {
$this->info('Truncating tables...');
try {
// First delete retribution proposals that reference spatial plannings
$deletedProposals = DB::table('retribution_proposals')
->whereNotNull('spatial_planning_id')
->count();
if ($deletedProposals > 0) {
DB::table('retribution_proposals')
->whereNotNull('spatial_planning_id')
->delete();
$this->info("Deleted {$deletedProposals} retribution proposals linked to spatial plannings.");
}
// Method 1: Try truncate with disabled foreign key checks
try {
DB::statement('SET FOREIGN_KEY_CHECKS = 0');
DB::table('spatial_plannings')->truncate();
DB::statement('SET FOREIGN_KEY_CHECKS = 1');
$this->info('Spatial plannings table truncated successfully.');
} catch (\Exception $truncateError) {
// Method 2: Fallback to delete if truncate fails
$this->warn('Truncate failed, using delete method...');
$deletedSpatial = DB::table('spatial_plannings')->delete();
$this->info("Deleted {$deletedSpatial} spatial planning records.");
// Reset auto increment
DB::statement('ALTER TABLE spatial_plannings AUTO_INCREMENT = 1');
}
} catch (\Exception $e) {
$this->error('Failed to truncate tables: ' . $e->getMessage());
return 1;
}
} else {
$this->info('Operation cancelled.');
return 0;
}
}
// Force truncate - disable foreign key checks and truncate everything
if ($this->option('force-truncate')) {
if ($this->confirm('This will FORCE truncate ALL spatial planning data by disabling foreign key checks. This is risky! Continue?')) {
$this->info('Force truncating with disabled foreign key checks...');
try {
DB::beginTransaction();
// Disable foreign key checks
DB::statement('SET FOREIGN_KEY_CHECKS = 0');
// Truncate both tables
DB::table('retribution_proposals')->truncate();
DB::table('spatial_plannings')->truncate();
// Re-enable foreign key checks
DB::statement('SET FOREIGN_KEY_CHECKS = 1');
$this->info('Force truncate completed successfully.');
DB::commit();
} catch (\Exception $e) {
DB::rollBack();
// Make sure to re-enable foreign key checks even on error
try {
DB::statement('SET FOREIGN_KEY_CHECKS = 1');
} catch (\Exception $fkError) {
$this->error('Failed to re-enable foreign key checks: ' . $fkError->getMessage());
}
$this->error('Failed to force truncate: ' . $e->getMessage());
return 1;
}
} else {
$this->info('Operation cancelled.');
return 0;
}
}
// Safe truncate - only delete spatial plannings without retribution proposals
if ($this->option('safe-truncate')) {
if ($this->confirm('This will delete only spatial planning data that have no retribution proposals. Continue?')) {
$this->info('Safe truncating spatial plannings...');
try {
DB::beginTransaction();
// Count spatial plannings with retribution proposals
$withProposals = DB::table('spatial_plannings')
->whereExists(function ($query) {
$query->select(DB::raw(1))
->from('retribution_proposals')
->whereColumn('retribution_proposals.spatial_planning_id', 'spatial_plannings.id');
})
->count();
// Delete spatial plannings without retribution proposals
$deletedCount = DB::table('spatial_plannings')
->whereNotExists(function ($query) {
$query->select(DB::raw(1))
->from('retribution_proposals')
->whereColumn('retribution_proposals.spatial_planning_id', 'spatial_plannings.id');
})
->delete();
$this->info("Deleted {$deletedCount} spatial plannings without retribution proposals.");
$this->info("Kept {$withProposals} spatial plannings that have retribution proposals.");
DB::commit();
} catch (\Exception $e) {
DB::rollBack();
$this->error('Failed to safe truncate: ' . $e->getMessage());
return 1;
}
} else {
$this->info('Operation cancelled.');
return 0;
}
}
$this->info("Starting import from: {$filePath}");
$this->info("Full path: {$fullPath}");
try {
DB::beginTransaction();
$data = Excel::toArray([], $fullPath);
if (empty($data) || empty($data[0])) {
$this->error('No data found in the file.');
return 1;
}
$rows = $data[0]; // Get first sheet
$headers = array_shift($rows); // Remove header row
$this->info("Found " . count($rows) . " data rows to import.");
$this->info("Headers: " . implode(', ', $headers));
dd($rows[0]);
$progressBar = $this->output->createProgressBar(count($rows));
$progressBar->start();
$imported = 0;
$skipped = 0;
foreach ($rows as $index => $row) {
try {
// Skip empty rows
if (empty(array_filter($row))) {
$skipped++;
$progressBar->advance();
continue;
}
// Map CSV columns to model attributes
$spatialData = [
'name' => $this->cleanString($row[1] ?? ''), // pemohon
'location' => $this->cleanString($row[2] ?? ''), // alamat
'activities' => $this->cleanString($row[3] ?? ''), // activities
'land_area' => $this->cleanNumber($row[4] ?? 0), // luas_lahan
'site_bcr' => $this->cleanNumber($row[5] ?? 0), // bcr_kawasan
'area' => $this->cleanNumber($row[6] ?? 0), // area
'no_tapak' => $this->cleanString($row[7] ?? ''), // no_tapak
'no_skkl' => $this->cleanString($row[8] ?? ''), // no_skkl
'no_ukl' => $this->cleanString($row[9] ?? ''), // no_ukl
'building_function' => $this->cleanString($row[10] ?? ''), // fungsi_bangunan
'sub_building_function' => $this->cleanString($row[11] ?? ''), // sub_fungsi_bangunan
'number_of_floors' => $this->cleanNumber($row[12] ?? 1), // jumlah_lantai
'number' => $this->cleanString($row[0] ?? ''), // no
'date' => now(), // Set current date
'kbli' => null, // Not in CSV, set as null
];
// Validate required fields
if (empty($spatialData['name']) && empty($spatialData['activities'])) {
$skipped++;
$progressBar->advance();
continue;
}
SpatialPlanning::create($spatialData);
$imported++;
} catch (Exception $e) {
$this->newLine();
$this->error("Error importing row " . ($index + 2) . ": " . $e->getMessage());
$skipped++;
}
$progressBar->advance();
}
$progressBar->finish();
$this->newLine(2);
DB::commit();
$this->info("Import completed successfully!");
$this->info("Imported: {$imported} records");
$this->info("Skipped: {$skipped} records");
return 0;
} catch (Exception $e) {
DB::rollBack();
$this->error("Import failed: " . $e->getMessage());
return 1;
}
}
/**
* Clean string data
*/
private function cleanString($value)
{
if (is_null($value)) return null;
return trim(str_replace(["\n", "\r", "\t"], ' ', $value));
}
/**
* Clean numeric data
*/
private function cleanNumber($value)
{
if (is_null($value) || $value === '') return 0;
// Remove non-numeric characters except decimal point
$cleaned = preg_replace('/[^0-9.]/', '', $value);
return is_numeric($cleaned) ? (float) $cleaned : 0;
}
/**
* List available template files
*/
private function listAvailableFiles()
{
$templatesPath = storage_path('app/public/templates');
if (is_dir($templatesPath)) {
$this->info("Files in storage/app/public/templates:");
$extensions = ['csv', 'xlsx', 'xls'];
foreach ($extensions as $ext) {
$files = glob($templatesPath . '/*.' . $ext);
foreach ($files as $file) {
$this->line(' - ' . basename($file));
}
}
}
$publicTemplatesPath = public_path('templates');
if (is_dir($publicTemplatesPath)) {
$this->info("Files in public/templates:");
$extensions = ['csv', 'xlsx', 'xls'];
foreach ($extensions as $ext) {
$files = glob($publicTemplatesPath . '/*.' . $ext);
foreach ($files as $file) {
$this->line(' - ' . basename($file));
}
}
}
}
}

45
app/Console/Commands/ScrapingData.php
View File

@@ -1,45 +0,0 @@
<?php
namespace App\Console\Commands;
use App\Jobs\ScrapingDataJob;
use App\Models\ImportDatasource;
use App\Services\ServiceGoogleSheet;
use App\Services\ServicePbgTask;
use App\Services\ServiceTabPbgTask;
use App\Services\ServiceTokenSIMBG;
use GuzzleHttp\Client; // Import Guzzle Client
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\DB;
class ScrapingData extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'app:scraping-data';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Command description';
/**
* Inject dependencies.
*/
public function __construct(
) {
parent::__construct();
}
public function handle()
{
dispatch(new ScrapingDataJob());
$this->info("Scraping job dispatched successfully");
}
}

80
app/Console/Commands/StartScrapingData.php Normal file
View File

@@ -0,0 +1,80 @@
<?php
namespace App\Console\Commands;
use App\Jobs\ScrapingDataJob;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Log;
class StartScrapingData extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'app:start-scraping-data
{--confirm : Skip confirmation prompt}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Start the optimized scraping data job (Google Sheet -> PBG Task -> Details)';
/**
* Execute the console command.
*/
public function handle()
{
$this->info('🚀 Starting Optimized Scraping Data Job');
$this->info('=====================================');
if (!$this->option('confirm')) {
$this->warn('⚠️ This will start a comprehensive data scraping process:');
$this->line(' 1. Google Sheet data scraping');
$this->line(' 2. PBG Task parent data scraping');
$this->line(' 3. Detailed task information scraping');
$this->line(' 4. BigData resume generation');
$this->newLine();
if (!$this->confirm('Do you want to continue?')) {
$this->info('Operation cancelled.');
return 0;
}
}
try {
// Dispatch the optimized job
$job = new ScrapingDataJob();
dispatch($job);
Log::info('ScrapingDataJob dispatched via command', [
'command' => $this->signature,
'user' => $this->option('confirm') ? 'auto' : 'manual'
]);
$this->info('✅ Scraping Data Job has been dispatched to the scraping queue!');
$this->newLine();
$this->info('📊 Monitor the job with:');
$this->line(' php artisan queue:monitor scraping');
$this->newLine();
$this->info('📜 View detailed logs with:');
$this->line(' tail -f /var/log/supervisor/sibedas-queue-scraping.log | grep "SCRAPING DATA JOB"');
$this->newLine();
$this->info('🔍 Check ImportDatasource status:');
$this->line(' docker-compose -f docker-compose.local.yml exec app php artisan tinker --execute="App\\Models\\ImportDatasource::latest()->first();"');
} catch (\Exception $e) {
$this->error('❌ Failed to dispatch ScrapingDataJob: ' . $e->getMessage());
Log::error('Failed to dispatch ScrapingDataJob via command', [
'error' => $e->getMessage(),
'trace' => $e->getTraceAsString()
]);
return 1;
}
return 0;
}
}

31
app/Http/Controllers/RequestAssignment/PbgTaskController.php
View File

@@ -56,10 +56,37 @@ class PbgTaskController extends Controller
*/
public function show(string $id)
{
$data = PbgTask::with(['pbg_task_retributions','pbg_task_index_integrations', 'pbg_task_retributions.pbg_task_prasarana', 'pbg_task_detail'])->findOrFail($id);
$data = PbgTask::with([
'pbg_task_retributions',
'pbg_task_index_integrations',
'pbg_task_retributions.pbg_task_prasarana',
'pbg_task_detail',
'dataLists' => function($query) {
$query->orderBy('data_type')->orderBy('name');
}
])->findOrFail($id);
// Group data lists by data_type for easier display
$dataListsByType = $data->dataLists->groupBy('data_type');
// Debug: Log the data types found for this task
\Log::info('PBG Task Data Lists', [
'task_uuid' => $data->uuid,
'total_data_lists' => $data->dataLists->count(),
'data_types_found' => $dataListsByType->keys()->toArray(),
'data_types_with_names' => $dataListsByType->map(function($items, $type) {
return [
'type' => $type,
'name' => $items->first()->data_type_name ?? "Type {$type}",
'count' => $items->count()
];
})->values()->toArray()
]);
$statusOptions = PbgTaskStatus::getStatuses();
$applicationTypes = PbgTaskApplicationTypes::labels();
return view("pbg_task.show", compact("data", 'statusOptions', 'applicationTypes'));
return view("pbg_task.show", compact("data", 'statusOptions', 'applicationTypes', 'dataListsByType'));
}
/**

3
app/Jobs/RetrySyncronizeJob.php
View File

@@ -48,8 +48,7 @@ class RetrySyncronizeJob implements ShouldQueue
$data_setting_result = $service_google_sheet->get_big_resume_data();
BigdataResume::generateResumeData($failed_import->id, "all", $data_setting_result);
BigdataResume::generateResumeData($failed_import->id, now()->year, $data_setting_result);
BigdataResume::generateResumeData($failed_import->id, "simbg", $data_setting_result);
$failed_import->update([
'status' => ImportDatasourceStatus::Success->value,

195
app/Jobs/ScrapingDataJob.php
View File

@@ -4,6 +4,7 @@ namespace App\Jobs;
use App\Models\BigdataResume;
use App\Models\ImportDatasource;
use App\Models\PbgTask;
use App\Services\ServiceGoogleSheet;
use App\Services\ServicePbgTask;
use App\Services\ServiceTabPbgTask;
@@ -21,72 +22,208 @@ class ScrapingDataJob implements ShouldQueue
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
/**
* Inject dependencies instead of creating them inside.
* Create a new job instance.
*/
public function __construct(
) {
public function __construct()
{
// Use dedicated scraping queue
$this->queue = 'scraping';
}
/**
* Execute the job.
* Execute the job with optimized schema:
* 1. Scrape Google Sheet first
* 2. Scrape PBG Task to get parent data
* 3. Loop through parent tasks to scrape details via ServiceTabPbgTask
*/
public function handle()
{
$import_datasource = null;
$failed_uuid = null;
try {
$processedTasks = 0;
$totalTasks = 0;
$client = app(Client::class);
try {
Log::info("=== SCRAPING DATA JOB STARTED ===");
// Initialize services
$service_google_sheet = app(ServiceGoogleSheet::class);
$service_pbg_task = app(ServicePbgTask::class);
$service_tab_pbg_task = app(ServiceTabPbgTask::class);
$service_google_sheet = app(ServiceGoogleSheet::class);
$service_token = app(ServiceTokenSIMBG::class);
// Create a record with "processing" status
// Create ImportDatasource record
$import_datasource = ImportDatasource::create([
'message' => 'Initiating scraping...',
'message' => 'Starting optimized scraping process...',
'response_body' => null,
'status' => 'processing',
'start_time' => now(),
'failed_uuid' => null
]);
// Run the scraping services
Log::info("ImportDatasource created", ['id' => $import_datasource->id]);
// STEP 1: Scrape Google Sheet data first
Log::info("=== STEP 1: SCRAPING GOOGLE SHEET ===");
$import_datasource->update(['message' => 'Scraping Google Sheet data...']);
$service_google_sheet->run_service();
Log::info("Google Sheet scraping completed successfully");
// STEP 2: Scrape PBG Task to get parent data
Log::info("=== STEP 2: SCRAPING PBG TASK PARENT DATA ===");
$import_datasource->update(['message' => 'Scraping PBG Task parent data...']);
$service_pbg_task->run_service();
try{
$service_tab_pbg_task->run_service();
}catch(\Exception $e){
$failed_uuid = $service_tab_pbg_task->getFailedUUID();
throw $e;
}
Log::info("PBG Task parent data scraping completed");
// STEP 3: Get all PBG tasks for detail scraping
$totalTasks = PbgTask::count();
Log::info("=== STEP 3: SCRAPING PBG TASK DETAILS ===", [
'total_tasks' => $totalTasks
]);
$import_datasource->update([
'message' => "Scraping details for {$totalTasks} PBG tasks..."
]);
// Process tasks in chunks for memory efficiency
$chunkSize = 100;
$processedTasks = 0;
PbgTask::orderBy('id')->chunk($chunkSize, function ($pbg_tasks) use (
$service_tab_pbg_task,
&$processedTasks,
$totalTasks,
$import_datasource,
&$failed_uuid
) {
foreach ($pbg_tasks as $pbg_task) {
try {
// Scrape all details for this task
$this->processTaskDetails($service_tab_pbg_task, $pbg_task->uuid);
$processedTasks++;
// Update progress every 10 tasks
if ($processedTasks % 10 === 0) {
$progress = round(($processedTasks / $totalTasks) * 100, 2);
Log::info("Progress update", [
'processed' => $processedTasks,
'total' => $totalTasks,
'progress' => "{$progress}%"
]);
$import_datasource->update([
'message' => "Processing details: {$processedTasks}/{$totalTasks} ({$progress}%)"
]);
}
} catch (\Exception $e) {
Log::warning("Failed to process task details", [
'uuid' => $pbg_task->uuid,
'error' => $e->getMessage()
]);
// Store failed UUID but continue processing
$failed_uuid = $pbg_task->uuid;
// Only stop if it's a critical error
if ($this->isCriticalError($e)) {
throw $e;
}
}
}
});
Log::info("Task details scraping completed", [
'processed_tasks' => $processedTasks,
'total_tasks' => $totalTasks
]);
// STEP 4: Generate BigData Resume
Log::info("=== STEP 4: GENERATING BIGDATA RESUME ===");
$import_datasource->update(['message' => 'Generating BigData resume...']);
$data_setting_result = $service_google_sheet->get_big_resume_data();
BigdataResume::generateResumeData($import_datasource->id, "simbg", $data_setting_result);
Log::info("BigData resume generated successfully");
// BigdataResume::generateResumeData($import_datasource->id, "all", $data_setting_result);
BigdataResume::generateResumeData($import_datasource->id, now()->year, $data_setting_result);
// Update status to success
// Update final status
$import_datasource->update([
'status' => 'success',
'message' => 'Scraping completed successfully.',
'finish_time' => now()
'message' => "Scraping completed successfully. Processed {$processedTasks}/{$totalTasks} tasks.",
'finish_time' => now(),
'failed_uuid' => $failed_uuid // Store last failed UUID if any
]);
Log::info("=== SCRAPING DATA JOB COMPLETED SUCCESSFULLY ===", [
'import_datasource_id' => $import_datasource->id,
'processed_tasks' => $processedTasks,
'total_tasks' => $totalTasks,
'has_failures' => !is_null($failed_uuid)
]);
} catch (\Exception $e) {
Log::error('Scraping failed: ' . $e->getMessage(), ['trace' => $e->getTraceAsString()]);
Log::error('=== SCRAPING DATA JOB FAILED ===', [
'error' => $e->getMessage(),
'file' => $e->getFile(),
'line' => $e->getLine(),
'processed_tasks' => $processedTasks,
'total_tasks' => $totalTasks,
'failed_uuid' => $failed_uuid,
'trace' => $e->getTraceAsString()
]);
// Update status to failed
if (isset($import_datasource)) {
// Update ImportDatasource with failure info
if ($import_datasource) {
$import_datasource->update([
'status' => 'failed',
'message' => 'Terjadi kesalahan, Syncronize tidak selesai',
'response_body' => 'Terjadi kesalahan, Syncronize tidak selesai',
'message' => "Scraping failed: {$e->getMessage()}. Processed {$processedTasks}/{$totalTasks} tasks.",
'response_body' => 'Scraping process interrupted due to error',
'finish_time' => now(),
'failed_uuid' => $failed_uuid,
]);
}
// Mark the job as failed
// Don't retry this job
$this->fail($e);
}
}
/**
* Process all detail endpoints for a single PBG task
*/
private function processTaskDetails(ServiceTabPbgTask $service, string $uuid): void
{
// Call all detail scraping methods for this task
$service->scraping_task_details($uuid);
$service->scraping_pbg_data_list($uuid);
$service->scraping_task_retributions($uuid);
$service->scraping_task_integrations($uuid);
}
/**
* Determine if an error is critical enough to stop the entire process
*/
private function isCriticalError(\Exception $e): bool
{
$criticalMessages = [
'authentication failed',
'token expired',
'database connection',
'memory exhausted',
'maximum execution time'
];
$errorMessage = strtolower($e->getMessage());
foreach ($criticalMessages as $critical) {
if (strpos($errorMessage, $critical) !== false) {
return true;
}
}
return false;
}
}

82
app/Models/PbgTask.php
View File

@@ -54,4 +54,86 @@ class PbgTask extends Model
public function attachments(){
return $this->hasMany(PbgTaskAttachment::class, 'pbg_task_id', 'id');
}
/**
* Get the data lists associated with this PBG task (One to Many)
* One pbg_task can have many data lists
*/
public function dataLists()
{
return $this->hasMany(PbgTaskDetailDataList::class, 'pbg_task_uuid', 'uuid');
}
/**
* Get only data lists with files
*/
public function dataListsWithFiles()
{
return $this->hasMany(PbgTaskDetailDataList::class, 'pbg_task_uuid', 'uuid')
->whereNotNull('file')
->where('file', '!=', '');
}
/**
* Get data lists by status
*/
public function dataListsByStatus($status)
{
return $this->hasMany(PbgTaskDetailDataList::class, 'pbg_task_uuid', 'uuid')
->where('status', $status);
}
/**
* Get data lists by data type
*/
public function dataListsByType($dataType)
{
return $this->hasMany(PbgTaskDetailDataList::class, 'pbg_task_uuid', 'uuid')
->where('data_type', $dataType);
}
/**
* Create or update data lists from API response
*/
public function syncDataLists(array $dataLists): void
{
foreach ($dataLists as $listData) {
PbgTaskDetailDataList::updateOrCreate(
['uid' => $listData['uid']],
[
'name' => $listData['name'] ?? null,
'description' => $listData['description'] ?? null,
'status' => $listData['status'] ?? null,
'status_name' => $listData['status_name'] ?? null,
'data_type' => $listData['data_type'] ?? null,
'data_type_name' => $listData['data_type_name'] ?? null,
'file' => $listData['file'] ?? null,
'note' => $listData['note'] ?? null,
'pbg_task_uuid' => $this->uuid,
]
);
}
}
/**
* Get data lists count by status
*/
public function getDataListsCountByStatusAttribute()
{
return $this->dataLists()
->selectRaw('status, COUNT(*) as count')
->groupBy('status')
->pluck('count', 'status');
}
/**
* Get data lists count by data type
*/
public function getDataListsCountByTypeAttribute()
{
return $this->dataLists()
->selectRaw('data_type, COUNT(*) as count')
->groupBy('data_type')
->pluck('count', 'data_type');
}
}

4
app/Models/PbgTaskDetail.php
View File

@@ -141,6 +141,8 @@ class PbgTaskDetail extends Model
return $this->belongsTo(PbgTask::class, 'pbg_task_uid', 'uuid');
}
/**
* Create or update PbgTaskDetail from API response
*/
@@ -249,4 +251,6 @@ class PbgTaskDetail extends Model
$detailData
);
}
}

161
app/Models/PbgTaskDetailDataList.php Normal file
View File

@@ -0,0 +1,161 @@
<?php
namespace App\Models;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Model;
class PbgTaskDetailDataList extends Model
{
use HasFactory;
protected $table = 'pbg_task_detail_data_lists';
protected $fillable = [
'uid',
'name',
'description',
'status',
'status_name',
'data_type',
'data_type_name',
'file',
'note',
'pbg_task_uuid',
];
protected $casts = [
'status' => 'integer',
'data_type' => 'integer',
'created_at' => 'datetime',
'updated_at' => 'datetime',
];
/**
* Relationship to PbgTask (Many to One)
* Many data lists belong to one pbg_task
*/
public function pbgTask()
{
return $this->belongsTo(PbgTask::class, 'pbg_task_uuid', 'uuid');
}
/**
* Get the full file path
*/
public function getFilePathAttribute()
{
return $this->file ? storage_path('app/public/' . $this->file) : null;
}
/**
* Get the file URL
*/
public function getFileUrlAttribute()
{
return $this->file ? asset('storage/' . $this->file) : null;
}
/**
* Check if file exists
*/
public function hasFile()
{
return !empty($this->file) && file_exists($this->getFilePathAttribute());
}
/**
* Get status badge color based on status
*/
public function getStatusBadgeAttribute()
{
return match($this->status) {
1 => 'success', // Sesuai
0 => 'danger', // Tidak Sesuai
default => 'secondary'
};
}
/**
* Scope: Filter by status
*/
public function scopeByStatus($query, $status)
{
return $query->where('status', $status);
}
/**
* Scope: Filter by data type
*/
public function scopeByDataType($query, $dataType)
{
return $query->where('data_type', $dataType);
}
/**
* Scope: With files only
*/
public function scopeWithFiles($query)
{
return $query->whereNotNull('file')->where('file', '!=', '');
}
/**
* Scope: Search by name or description
*/
public function scopeSearch($query, $search)
{
return $query->where(function ($q) use ($search) {
$q->where('name', 'LIKE', "%{$search}%")
->orWhere('description', 'LIKE', "%{$search}%")
->orWhere('status_name', 'LIKE', "%{$search}%")
->orWhere('data_type_name', 'LIKE', "%{$search}%");
});
}
/**
* Get file extension from file path
*/
public function getFileExtensionAttribute()
{
if (!$this->file) {
return null;
}
return strtoupper(pathinfo($this->file, PATHINFO_EXTENSION));
}
/**
* Get filename from file path
*/
public function getFileNameAttribute()
{
if (!$this->file) {
return null;
}
return basename($this->file);
}
/**
* Get formatted created date
*/
public function getFormattedCreatedAtAttribute()
{
return $this->created_at ? $this->created_at->format('d M Y, H:i') : '-';
}
/**
* Get truncated description
*/
public function getTruncatedDescriptionAttribute()
{
return $this->description ? \Str::limit($this->description, 80) : null;
}
/**
* Get truncated note
*/
public function getTruncatedNoteAttribute()
{
return $this->note ? \Str::limit($this->note, 100) : null;
}
}

2
app/Services/ServicePbgTask.php
View File

@@ -106,7 +106,7 @@ class ServicePbgTask
};
do {
$url = "{$this->simbg_host}/api/pbg/v1/list/?page={$currentPage}&size={$this->fetch_per_page}&sort=ASC&date&search&status&slf_status&type&sort_by=created_at&application_type=1&start_date&end_date";
$url = "{$this->simbg_host}/api/pbg/v1/list/?page={$currentPage}&size={$this->fetch_per_page}&sort=ASC&date&search&status&slf_status&type=task&sort_by=created_at&application_type=1&start_date&end_date";
$fetch_data = $fetchData($url);
if (!$fetch_data) {

290
app/Services/ServiceTabPbgTask.php
View File

@@ -5,6 +5,7 @@ namespace App\Services;
use App\Models\GlobalSetting;
use App\Models\PbgTask;
use App\Models\PbgTaskDetail;
use App\Models\PbgTaskDetailDataList;
use App\Models\PbgTaskIndexIntegrations;
use App\Models\PbgTaskPrasarana;
use App\Models\PbgTaskRetributions;
@@ -36,44 +37,119 @@ class ServiceTabPbgTask
$this->user_refresh_token = $auth_data['refresh'];
}
public function run_service($retry_uuid = null)
public function run_service($retry_uuid = null, $chunk_size = 50)
{
try {
$pbg_tasks = PbgTask::orderBy('id')->get();
$start = false;
$query = PbgTask::orderBy('id');
// If retry_uuid is provided, start from that UUID
if ($retry_uuid) {
$retryTask = PbgTask::where('uuid', $retry_uuid)->first();
if ($retryTask) {
$query->where('id', '>=', $retryTask->id);
Log::info("Resuming sync from UUID: {$retry_uuid} (ID: {$retryTask->id})");
}
}
foreach ($pbg_tasks as $pbg_task) {
if($retry_uuid){
if($pbg_task->uuid === $retry_uuid){
$start = true;
}
$totalTasks = $query->count();
$processedCount = 0;
Log::info("Starting sync for {$totalTasks} PBG Tasks with chunk size: {$chunk_size}");
if(!$start){
// Process in chunks to reduce memory usage
$query->chunk($chunk_size, function ($pbg_tasks) use (&$processedCount, $totalTasks) {
$chunkStartTime = now();
foreach ($pbg_tasks as $pbg_task) {
try {
$this->current_uuid = $pbg_task->uuid;
$taskStartTime = now();
// Process all endpoints for this task
$this->processTaskEndpoints($pbg_task->uuid);
$processedCount++;
$taskTime = now()->diffInSeconds($taskStartTime);
// Log progress every 10 tasks
if ($processedCount % 10 === 0) {
$progress = round(($processedCount / $totalTasks) * 100, 2);
Log::info("Progress: {$processedCount}/{$totalTasks} ({$progress}%) - Last task took {$taskTime}s");
}
} catch (\Exception $e) {
Log::error("Failed on UUID: {$this->current_uuid}, Error: " . $e->getMessage());
// Check if this is a critical error that should stop the process
if ($this->isCriticalError($e)) {
throw $e;
}
// For non-critical errors, log and continue
Log::warning("Skipping UUID {$this->current_uuid} due to non-critical error");
continue;
}
}
try{
$this->current_uuid = $pbg_task->uuid;
$this->scraping_task_details($pbg_task->uuid);
// $this->scraping_task_assignments($pbg_task->uuid);
$this->scraping_task_retributions($pbg_task->uuid);
$this->scraping_task_integrations($pbg_task->uuid);
}catch(\Exception $e){
Log::error("Failed on UUID: {$this->current_uuid}, Error: " . $e->getMessage());
throw $e;
$chunkTime = now()->diffInSeconds($chunkStartTime);
Log::info("Processed chunk of {$pbg_tasks->count()} tasks in {$chunkTime} seconds");
// Small delay between chunks to prevent API rate limiting
if ($pbg_tasks->count() === $chunk_size) {
sleep(1);
}
}
});
Log::info("Successfully completed sync for {$processedCount} PBG Tasks");
} catch (\Exception $e) {
Log::error("Failed to syncronize: " . $e->getMessage());
Log::error("Failed to synchronize: " . $e->getMessage());
throw $e;
}
}
/**
* Process all endpoints for a single task
*/
private function processTaskEndpoints(string $uuid): void
{
$this->scraping_task_details($uuid);
$this->scraping_pbg_data_list($uuid);
// $this->scraping_task_assignments($uuid);
$this->scraping_task_retributions($uuid);
$this->scraping_task_integrations($uuid);
}
/**
* Determine if an error is critical and should stop the process
*/
private function isCriticalError(\Exception $e): bool
{
$message = $e->getMessage();
// Critical authentication errors
if (strpos($message, 'Token refresh and login failed') !== false) {
return true;
}
// Critical system errors
if (strpos($message, 'Connection refused') !== false) {
return true;
}
// Database connection errors
if (strpos($message, 'database') !== false && strpos($message, 'connection') !== false) {
return true;
}
return false;
}
public function getFailedUUID(){
return $this->current_uuid;
}
private function scraping_task_details($uuid)
public function scraping_task_details($uuid)
{
$url = "{$this->simbg_host}/api/pbg/v1/detail/{$uuid}/";
$options = [
@@ -144,7 +220,7 @@ class ServiceTabPbgTask
throw new \Exception("Failed to fetch task details for UUID {$uuid} after retries.");
}
private function scraping_task_assignments($uuid)
public function scraping_task_assignments($uuid)
{
$url = "{$this->simbg_host}/api/pbg/v1/list-tim-penilai/{$uuid}/?page=1&size=10";
$options = [
@@ -237,7 +313,173 @@ class ServiceTabPbgTask
throw new \Exception("Failed to fetch task assignments for UUID {$uuid} after retries.");
}
private function scraping_task_retributions($uuid)
public function scraping_pbg_data_list($uuid){
$url = "{$this->simbg_host}/api/pbg/v1/detail/{$uuid}/list-data/?sort=DESC";
$options = [
'headers' => [
'Authorization' => "Bearer {$this->user_token}",
'Content-Type' => 'application/json'
]
];
$maxRetries = 3;
$initialDelay = 1;
$retriedAfter401 = false;
for ($retryCount = 0; $retryCount < $maxRetries; $retryCount++) {
try{
$response = $this->client->get($url, $options);
$responseData = json_decode($response->getBody()->getContents(), true, 512, JSON_THROW_ON_ERROR);
if (empty($responseData['data']) || !is_array($responseData['data'])) {
Log::info("No data list found for UUID: {$uuid}");
return true;
}
$data = $responseData['data'];
Log::info("Processing data list for UUID: {$uuid}, found " . count($data) . " items");
// Process each data list item and save to database
$this->processDataListItems($data, $uuid);
return $responseData;
} catch (\GuzzleHttp\Exception\ClientException $e) {
if ($e->getCode() === 401 && !$retriedAfter401) {
Log::warning("401 Unauthorized - Refreshing token and retrying...");
try{
$this->refreshToken();
$options['headers']['Authorization'] = "Bearer {$this->user_token}";
$retriedAfter401 = true;
continue;
}catch(\Exception $refreshError){
Log::error("Token refresh and login failed: " . $refreshError->getMessage());
return false;
}
}
return false;
} catch (\GuzzleHttp\Exception\ServerException | \GuzzleHttp\Exception\ConnectException $e) {
if ($e->getCode() === 502) {
Log::warning("502 Bad Gateway - Retrying in {$initialDelay} seconds...");
} else {
Log::error("Network error ({$e->getCode()}) - Retrying in {$initialDelay} seconds...");
}
sleep($initialDelay);
$initialDelay *= 2;
} catch (\GuzzleHttp\Exception\RequestException $e) {
Log::error("Request error ({$e->getCode()}): " . $e->getMessage());
return false;
} catch (\JsonException $e) {
Log::error("JSON decoding error: " . $e->getMessage());
return false;
} catch (\Throwable $e) {
Log::critical("Unhandled error: " . $e->getMessage(), ['trace' => $e->getTraceAsString()]);
return false;
}
}
Log::error("Failed to fetch task data list for UUID {$uuid} after {$maxRetries} retries.");
throw new \Exception("Failed to fetch task data list for UUID {$uuid} after retries.");
}
/**
* Process and save data list items to database (Optimized with bulk operations)
*/
private function processDataListItems(array $dataListItems, string $pbgTaskUuid): void
{
try {
if (empty($dataListItems)) {
return;
}
$batchData = [];
$validItems = 0;
foreach ($dataListItems as $item) {
// Validate required fields
if (empty($item['uid'])) {
Log::warning("Skipping data list item with missing UID for PBG Task: {$pbgTaskUuid}");
continue;
}
// Parse created_at if exists
$createdAt = null;
if (!empty($item['created_at'])) {
try {
$createdAt = Carbon::parse($item['created_at'])->format('Y-m-d H:i:s');
} catch (\Exception $e) {
Log::warning("Invalid created_at format for data list UID: {$item['uid']}, Error: " . $e->getMessage());
}
}
$batchData[] = [
'uid' => $item['uid'],
'name' => $item['name'] ?? null,
'description' => $item['description'] ?? null,
'status' => $item['status'] ?? null,
'status_name' => $item['status_name'] ?? null,
'data_type' => $item['data_type'] ?? null,
'data_type_name' => $item['data_type_name'] ?? null,
'file' => $item['file'] ?? null,
'note' => $item['note'] ?? null,
'pbg_task_uuid' => $pbgTaskUuid,
'created_at' => $createdAt ?: now(),
'updated_at' => now(),
];
$validItems++;
}
if (!empty($batchData)) {
// Use upsert for bulk insert/update operations
PbgTaskDetailDataList::upsert(
$batchData,
['uid'], // Unique columns
[
'name', 'description', 'status', 'status_name',
'data_type', 'data_type_name', 'file', 'note',
'pbg_task_uuid', 'updated_at'
] // Columns to update
);
Log::info("Successfully bulk processed {$validItems} data list items for PBG Task: {$pbgTaskUuid}");
}
} catch (\Exception $e) {
Log::error("Error bulk processing data list items for PBG Task {$pbgTaskUuid}: " . $e->getMessage());
throw $e;
}
}
/**
* Alternative method using PbgTask model's syncDataLists for cleaner code
*/
private function processDataListItemsWithModel(array $dataListItems, string $pbgTaskUuid): void
{
try {
// Find the PbgTask
$pbgTask = PbgTask::where('uuid', $pbgTaskUuid)->first();
if (!$pbgTask) {
Log::error("PBG Task not found with UUID: {$pbgTaskUuid}");
return;
}
// Use the model's syncDataLists method
$pbgTask->syncDataLists($dataListItems);
$processedCount = count($dataListItems);
Log::info("Successfully synced {$processedCount} data list items for PBG Task: {$pbgTaskUuid} using model method");
} catch (\Exception $e) {
Log::error("Error syncing data list items for PBG Task {$pbgTaskUuid}: " . $e->getMessage());
throw $e;
}
}
public function scraping_task_retributions($uuid)
{
$url = "{$this->simbg_host}/api/pbg/v1/detail/" . $uuid . "/retribution/submit/";
$options = [
@@ -354,7 +596,7 @@ class ServiceTabPbgTask
throw new \Exception("Failed to fetch task retributions for UUID {$uuid} after retries.");
}
private function scraping_task_integrations($uuid){
public function scraping_task_integrations($uuid){
$url = "{$this->simbg_host}/api/pbg/v1/detail/" . $uuid . "/retribution/indeks-terintegrasi/";
$options = [
'headers' => [