add new url scraping data and create tab data lists
This commit is contained in:
arifal
@@ -1,319 +0,0 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Models\SpatialPlanning;
|
||||
use Illuminate\Console\Command;
|
||||
use Maatwebsite\Excel\Facades\Excel;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Exception;
|
||||
|
||||
class InitSpatialPlanningDatas extends Command
|
||||
{
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'spatial:init {file? : Path to the CSV/Excel file} {--truncate : Clear existing data before import} {--safe-truncate : Clear only spatial plannings without retribution proposals} {--force-truncate : Force truncate by disabling foreign key checks}';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Import spatial planning data from CSV/Excel file for retribution';
|
||||
|
||||
/**
|
||||
* Execute the console command.
|
||||
*/
|
||||
public function handle()
|
||||
{
|
||||
$filePath = $this->argument('file') ?? 'public/templates/2025.xlsx';
|
||||
$fullPath = storage_path('app/' . $filePath);
|
||||
|
||||
// Check if file exists
|
||||
if (!file_exists($fullPath)) {
|
||||
$this->error("File not found: {$fullPath}");
|
||||
$this->info("Available files in templates:");
|
||||
$this->listAvailableFiles();
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Handle truncate options
|
||||
if (($this->option('truncate') && $this->option('safe-truncate')) ||
|
||||
($this->option('truncate') && $this->option('force-truncate')) ||
|
||||
($this->option('safe-truncate') && $this->option('force-truncate'))) {
|
||||
$this->error('Cannot use multiple truncate options together. Choose only one.');
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Confirm truncate if requested
|
||||
if ($this->option('truncate')) {
|
||||
if ($this->confirm('This will delete all existing spatial planning data and related retribution proposals. Continue?')) {
|
||||
$this->info('Truncating tables...');
|
||||
|
||||
try {
|
||||
// First delete retribution proposals that reference spatial plannings
|
||||
$deletedProposals = DB::table('retribution_proposals')
|
||||
->whereNotNull('spatial_planning_id')
|
||||
->count();
|
||||
|
||||
if ($deletedProposals > 0) {
|
||||
DB::table('retribution_proposals')
|
||||
->whereNotNull('spatial_planning_id')
|
||||
->delete();
|
||||
$this->info("Deleted {$deletedProposals} retribution proposals linked to spatial plannings.");
|
||||
}
|
||||
|
||||
// Method 1: Try truncate with disabled foreign key checks
|
||||
try {
|
||||
DB::statement('SET FOREIGN_KEY_CHECKS = 0');
|
||||
DB::table('spatial_plannings')->truncate();
|
||||
DB::statement('SET FOREIGN_KEY_CHECKS = 1');
|
||||
$this->info('Spatial plannings table truncated successfully.');
|
||||
} catch (\Exception $truncateError) {
|
||||
// Method 2: Fallback to delete if truncate fails
|
||||
$this->warn('Truncate failed, using delete method...');
|
||||
$deletedSpatial = DB::table('spatial_plannings')->delete();
|
||||
$this->info("Deleted {$deletedSpatial} spatial planning records.");
|
||||
|
||||
// Reset auto increment
|
||||
DB::statement('ALTER TABLE spatial_plannings AUTO_INCREMENT = 1');
|
||||
}
|
||||
|
||||
} catch (\Exception $e) {
|
||||
$this->error('Failed to truncate tables: ' . $e->getMessage());
|
||||
return 1;
|
||||
}
|
||||
} else {
|
||||
$this->info('Operation cancelled.');
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
// Force truncate - disable foreign key checks and truncate everything
|
||||
if ($this->option('force-truncate')) {
|
||||
if ($this->confirm('This will FORCE truncate ALL spatial planning data by disabling foreign key checks. This is risky! Continue?')) {
|
||||
$this->info('Force truncating with disabled foreign key checks...');
|
||||
|
||||
try {
|
||||
DB::beginTransaction();
|
||||
|
||||
// Disable foreign key checks
|
||||
DB::statement('SET FOREIGN_KEY_CHECKS = 0');
|
||||
|
||||
// Truncate both tables
|
||||
DB::table('retribution_proposals')->truncate();
|
||||
DB::table('spatial_plannings')->truncate();
|
||||
|
||||
// Re-enable foreign key checks
|
||||
DB::statement('SET FOREIGN_KEY_CHECKS = 1');
|
||||
|
||||
$this->info('Force truncate completed successfully.');
|
||||
|
||||
DB::commit();
|
||||
|
||||
} catch (\Exception $e) {
|
||||
DB::rollBack();
|
||||
// Make sure to re-enable foreign key checks even on error
|
||||
try {
|
||||
DB::statement('SET FOREIGN_KEY_CHECKS = 1');
|
||||
} catch (\Exception $fkError) {
|
||||
$this->error('Failed to re-enable foreign key checks: ' . $fkError->getMessage());
|
||||
}
|
||||
$this->error('Failed to force truncate: ' . $e->getMessage());
|
||||
return 1;
|
||||
}
|
||||
} else {
|
||||
$this->info('Operation cancelled.');
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
// Safe truncate - only delete spatial plannings without retribution proposals
|
||||
if ($this->option('safe-truncate')) {
|
||||
if ($this->confirm('This will delete only spatial planning data that have no retribution proposals. Continue?')) {
|
||||
$this->info('Safe truncating spatial plannings...');
|
||||
|
||||
try {
|
||||
DB::beginTransaction();
|
||||
|
||||
// Count spatial plannings with retribution proposals
|
||||
$withProposals = DB::table('spatial_plannings')
|
||||
->whereExists(function ($query) {
|
||||
$query->select(DB::raw(1))
|
||||
->from('retribution_proposals')
|
||||
->whereColumn('retribution_proposals.spatial_planning_id', 'spatial_plannings.id');
|
||||
})
|
||||
->count();
|
||||
|
||||
// Delete spatial plannings without retribution proposals
|
||||
$deletedCount = DB::table('spatial_plannings')
|
||||
->whereNotExists(function ($query) {
|
||||
$query->select(DB::raw(1))
|
||||
->from('retribution_proposals')
|
||||
->whereColumn('retribution_proposals.spatial_planning_id', 'spatial_plannings.id');
|
||||
})
|
||||
->delete();
|
||||
|
||||
$this->info("Deleted {$deletedCount} spatial plannings without retribution proposals.");
|
||||
$this->info("Kept {$withProposals} spatial plannings that have retribution proposals.");
|
||||
|
||||
DB::commit();
|
||||
|
||||
} catch (\Exception $e) {
|
||||
DB::rollBack();
|
||||
$this->error('Failed to safe truncate: ' . $e->getMessage());
|
||||
return 1;
|
||||
}
|
||||
} else {
|
||||
$this->info('Operation cancelled.');
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
$this->info("Starting import from: {$filePath}");
|
||||
$this->info("Full path: {$fullPath}");
|
||||
|
||||
try {
|
||||
DB::beginTransaction();
|
||||
|
||||
$data = Excel::toArray([], $fullPath);
|
||||
|
||||
if (empty($data) || empty($data[0])) {
|
||||
$this->error('No data found in the file.');
|
||||
return 1;
|
||||
}
|
||||
|
||||
$rows = $data[0]; // Get first sheet
|
||||
$headers = array_shift($rows); // Remove header row
|
||||
|
||||
$this->info("Found " . count($rows) . " data rows to import.");
|
||||
$this->info("Headers: " . implode(', ', $headers));
|
||||
|
||||
dd($rows[0]);
|
||||
|
||||
$progressBar = $this->output->createProgressBar(count($rows));
|
||||
$progressBar->start();
|
||||
|
||||
$imported = 0;
|
||||
$skipped = 0;
|
||||
|
||||
foreach ($rows as $index => $row) {
|
||||
try {
|
||||
// Skip empty rows
|
||||
if (empty(array_filter($row))) {
|
||||
$skipped++;
|
||||
$progressBar->advance();
|
||||
continue;
|
||||
}
|
||||
|
||||
// Map CSV columns to model attributes
|
||||
$spatialData = [
|
||||
'name' => $this->cleanString($row[1] ?? ''), // pemohon
|
||||
'location' => $this->cleanString($row[2] ?? ''), // alamat
|
||||
'activities' => $this->cleanString($row[3] ?? ''), // activities
|
||||
'land_area' => $this->cleanNumber($row[4] ?? 0), // luas_lahan
|
||||
'site_bcr' => $this->cleanNumber($row[5] ?? 0), // bcr_kawasan
|
||||
'area' => $this->cleanNumber($row[6] ?? 0), // area
|
||||
'no_tapak' => $this->cleanString($row[7] ?? ''), // no_tapak
|
||||
'no_skkl' => $this->cleanString($row[8] ?? ''), // no_skkl
|
||||
'no_ukl' => $this->cleanString($row[9] ?? ''), // no_ukl
|
||||
'building_function' => $this->cleanString($row[10] ?? ''), // fungsi_bangunan
|
||||
'sub_building_function' => $this->cleanString($row[11] ?? ''), // sub_fungsi_bangunan
|
||||
'number_of_floors' => $this->cleanNumber($row[12] ?? 1), // jumlah_lantai
|
||||
'number' => $this->cleanString($row[0] ?? ''), // no
|
||||
'date' => now(), // Set current date
|
||||
'kbli' => null, // Not in CSV, set as null
|
||||
];
|
||||
|
||||
// Validate required fields
|
||||
if (empty($spatialData['name']) && empty($spatialData['activities'])) {
|
||||
$skipped++;
|
||||
$progressBar->advance();
|
||||
continue;
|
||||
}
|
||||
|
||||
SpatialPlanning::create($spatialData);
|
||||
$imported++;
|
||||
|
||||
} catch (Exception $e) {
|
||||
$this->newLine();
|
||||
$this->error("Error importing row " . ($index + 2) . ": " . $e->getMessage());
|
||||
$skipped++;
|
||||
}
|
||||
|
||||
$progressBar->advance();
|
||||
}
|
||||
|
||||
$progressBar->finish();
|
||||
$this->newLine(2);
|
||||
|
||||
DB::commit();
|
||||
|
||||
$this->info("Import completed successfully!");
|
||||
$this->info("Imported: {$imported} records");
|
||||
$this->info("Skipped: {$skipped} records");
|
||||
|
||||
return 0;
|
||||
|
||||
} catch (Exception $e) {
|
||||
DB::rollBack();
|
||||
$this->error("Import failed: " . $e->getMessage());
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean string data
|
||||
*/
|
||||
private function cleanString($value)
|
||||
{
|
||||
if (is_null($value)) return null;
|
||||
return trim(str_replace(["\n", "\r", "\t"], ' ', $value));
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean numeric data
|
||||
*/
|
||||
private function cleanNumber($value)
|
||||
{
|
||||
if (is_null($value) || $value === '') return 0;
|
||||
|
||||
// Remove non-numeric characters except decimal point
|
||||
$cleaned = preg_replace('/[^0-9.]/', '', $value);
|
||||
|
||||
return is_numeric($cleaned) ? (float) $cleaned : 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* List available template files
|
||||
*/
|
||||
private function listAvailableFiles()
|
||||
{
|
||||
$templatesPath = storage_path('app/public/templates');
|
||||
if (is_dir($templatesPath)) {
|
||||
$this->info("Files in storage/app/public/templates:");
|
||||
$extensions = ['csv', 'xlsx', 'xls'];
|
||||
foreach ($extensions as $ext) {
|
||||
$files = glob($templatesPath . '/*.' . $ext);
|
||||
foreach ($files as $file) {
|
||||
$this->line(' - ' . basename($file));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$publicTemplatesPath = public_path('templates');
|
||||
if (is_dir($publicTemplatesPath)) {
|
||||
$this->info("Files in public/templates:");
|
||||
$extensions = ['csv', 'xlsx', 'xls'];
|
||||
foreach ($extensions as $ext) {
|
||||
$files = glob($publicTemplatesPath . '/*.' . $ext);
|
||||
foreach ($files as $file) {
|
||||
$this->line(' - ' . basename($file));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Jobs\ScrapingDataJob;
|
||||
use App\Models\ImportDatasource;
|
||||
use App\Services\ServiceGoogleSheet;
|
||||
use App\Services\ServicePbgTask;
|
||||
use App\Services\ServiceTabPbgTask;
|
||||
use App\Services\ServiceTokenSIMBG;
|
||||
use GuzzleHttp\Client; // Import Guzzle Client
|
||||
use Illuminate\Console\Command;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Illuminate\Support\Facades\DB;
|
||||
|
||||
class ScrapingData extends Command
|
||||
{
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'app:scraping-data';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Command description';
|
||||
/**
|
||||
* Inject dependencies.
|
||||
*/
|
||||
public function __construct(
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
public function handle()
|
||||
{
|
||||
dispatch(new ScrapingDataJob());
|
||||
|
||||
$this->info("Scraping job dispatched successfully");
|
||||
}
|
||||
}
|
||||
80
app/Console/Commands/StartScrapingData.php
Normal file
80
app/Console/Commands/StartScrapingData.php
Normal file
@@ -0,0 +1,80 @@
|
||||
<?php
|
||||
|
||||
namespace App\Console\Commands;
|
||||
|
||||
use App\Jobs\ScrapingDataJob;
|
||||
use Illuminate\Console\Command;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
|
||||
class StartScrapingData extends Command
|
||||
{
|
||||
/**
|
||||
* The name and signature of the console command.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $signature = 'app:start-scraping-data
|
||||
{--confirm : Skip confirmation prompt}';
|
||||
|
||||
/**
|
||||
* The console command description.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $description = 'Start the optimized scraping data job (Google Sheet -> PBG Task -> Details)';
|
||||
|
||||
/**
|
||||
* Execute the console command.
|
||||
*/
|
||||
public function handle()
|
||||
{
|
||||
$this->info('🚀 Starting Optimized Scraping Data Job');
|
||||
$this->info('=====================================');
|
||||
|
||||
if (!$this->option('confirm')) {
|
||||
$this->warn('⚠️ This will start a comprehensive data scraping process:');
|
||||
$this->line(' 1. Google Sheet data scraping');
|
||||
$this->line(' 2. PBG Task parent data scraping');
|
||||
$this->line(' 3. Detailed task information scraping');
|
||||
$this->line(' 4. BigData resume generation');
|
||||
$this->newLine();
|
||||
|
||||
if (!$this->confirm('Do you want to continue?')) {
|
||||
$this->info('Operation cancelled.');
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
// Dispatch the optimized job
|
||||
$job = new ScrapingDataJob();
|
||||
dispatch($job);
|
||||
|
||||
Log::info('ScrapingDataJob dispatched via command', [
|
||||
'command' => $this->signature,
|
||||
'user' => $this->option('confirm') ? 'auto' : 'manual'
|
||||
]);
|
||||
|
||||
$this->info('✅ Scraping Data Job has been dispatched to the scraping queue!');
|
||||
$this->newLine();
|
||||
$this->info('📊 Monitor the job with:');
|
||||
$this->line(' php artisan queue:monitor scraping');
|
||||
$this->newLine();
|
||||
$this->info('📜 View detailed logs with:');
|
||||
$this->line(' tail -f /var/log/supervisor/sibedas-queue-scraping.log | grep "SCRAPING DATA JOB"');
|
||||
$this->newLine();
|
||||
$this->info('🔍 Check ImportDatasource status:');
|
||||
$this->line(' docker-compose -f docker-compose.local.yml exec app php artisan tinker --execute="App\\Models\\ImportDatasource::latest()->first();"');
|
||||
|
||||
} catch (\Exception $e) {
|
||||
$this->error('❌ Failed to dispatch ScrapingDataJob: ' . $e->getMessage());
|
||||
Log::error('Failed to dispatch ScrapingDataJob via command', [
|
||||
'error' => $e->getMessage(),
|
||||
'trace' => $e->getTraceAsString()
|
||||
]);
|
||||
return 1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
@@ -56,10 +56,37 @@ class PbgTaskController extends Controller
|
||||
*/
|
||||
public function show(string $id)
|
||||
{
|
||||
$data = PbgTask::with(['pbg_task_retributions','pbg_task_index_integrations', 'pbg_task_retributions.pbg_task_prasarana', 'pbg_task_detail'])->findOrFail($id);
|
||||
$data = PbgTask::with([
|
||||
'pbg_task_retributions',
|
||||
'pbg_task_index_integrations',
|
||||
'pbg_task_retributions.pbg_task_prasarana',
|
||||
'pbg_task_detail',
|
||||
'dataLists' => function($query) {
|
||||
$query->orderBy('data_type')->orderBy('name');
|
||||
}
|
||||
])->findOrFail($id);
|
||||
|
||||
// Group data lists by data_type for easier display
|
||||
$dataListsByType = $data->dataLists->groupBy('data_type');
|
||||
|
||||
// Debug: Log the data types found for this task
|
||||
\Log::info('PBG Task Data Lists', [
|
||||
'task_uuid' => $data->uuid,
|
||||
'total_data_lists' => $data->dataLists->count(),
|
||||
'data_types_found' => $dataListsByType->keys()->toArray(),
|
||||
'data_types_with_names' => $dataListsByType->map(function($items, $type) {
|
||||
return [
|
||||
'type' => $type,
|
||||
'name' => $items->first()->data_type_name ?? "Type {$type}",
|
||||
'count' => $items->count()
|
||||
];
|
||||
})->values()->toArray()
|
||||
]);
|
||||
|
||||
$statusOptions = PbgTaskStatus::getStatuses();
|
||||
$applicationTypes = PbgTaskApplicationTypes::labels();
|
||||
return view("pbg_task.show", compact("data", 'statusOptions', 'applicationTypes'));
|
||||
|
||||
return view("pbg_task.show", compact("data", 'statusOptions', 'applicationTypes', 'dataListsByType'));
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -48,8 +48,7 @@ class RetrySyncronizeJob implements ShouldQueue
|
||||
|
||||
$data_setting_result = $service_google_sheet->get_big_resume_data();
|
||||
|
||||
BigdataResume::generateResumeData($failed_import->id, "all", $data_setting_result);
|
||||
BigdataResume::generateResumeData($failed_import->id, now()->year, $data_setting_result);
|
||||
BigdataResume::generateResumeData($failed_import->id, "simbg", $data_setting_result);
|
||||
|
||||
$failed_import->update([
|
||||
'status' => ImportDatasourceStatus::Success->value,
|
||||
|
||||
@@ -4,6 +4,7 @@ namespace App\Jobs;
|
||||
|
||||
use App\Models\BigdataResume;
|
||||
use App\Models\ImportDatasource;
|
||||
use App\Models\PbgTask;
|
||||
use App\Services\ServiceGoogleSheet;
|
||||
use App\Services\ServicePbgTask;
|
||||
use App\Services\ServiceTabPbgTask;
|
||||
@@ -21,72 +22,208 @@ class ScrapingDataJob implements ShouldQueue
|
||||
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||
|
||||
/**
|
||||
* Inject dependencies instead of creating them inside.
|
||||
* Create a new job instance.
|
||||
*/
|
||||
public function __construct(
|
||||
) {
|
||||
public function __construct()
|
||||
{
|
||||
// Use dedicated scraping queue
|
||||
$this->queue = 'scraping';
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the job.
|
||||
* Execute the job with optimized schema:
|
||||
* 1. Scrape Google Sheet first
|
||||
* 2. Scrape PBG Task to get parent data
|
||||
* 3. Loop through parent tasks to scrape details via ServiceTabPbgTask
|
||||
*/
|
||||
public function handle()
|
||||
{
|
||||
$import_datasource = null;
|
||||
$failed_uuid = null;
|
||||
try {
|
||||
$processedTasks = 0;
|
||||
$totalTasks = 0;
|
||||
|
||||
$client = app(Client::class);
|
||||
try {
|
||||
Log::info("=== SCRAPING DATA JOB STARTED ===");
|
||||
|
||||
// Initialize services
|
||||
$service_google_sheet = app(ServiceGoogleSheet::class);
|
||||
$service_pbg_task = app(ServicePbgTask::class);
|
||||
$service_tab_pbg_task = app(ServiceTabPbgTask::class);
|
||||
$service_google_sheet = app(ServiceGoogleSheet::class);
|
||||
$service_token = app(ServiceTokenSIMBG::class);
|
||||
// Create a record with "processing" status
|
||||
|
||||
// Create ImportDatasource record
|
||||
$import_datasource = ImportDatasource::create([
|
||||
'message' => 'Initiating scraping...',
|
||||
'message' => 'Starting optimized scraping process...',
|
||||
'response_body' => null,
|
||||
'status' => 'processing',
|
||||
'start_time' => now(),
|
||||
'failed_uuid' => null
|
||||
]);
|
||||
|
||||
// Run the scraping services
|
||||
Log::info("ImportDatasource created", ['id' => $import_datasource->id]);
|
||||
|
||||
// STEP 1: Scrape Google Sheet data first
|
||||
Log::info("=== STEP 1: SCRAPING GOOGLE SHEET ===");
|
||||
$import_datasource->update(['message' => 'Scraping Google Sheet data...']);
|
||||
|
||||
$service_google_sheet->run_service();
|
||||
Log::info("Google Sheet scraping completed successfully");
|
||||
|
||||
// STEP 2: Scrape PBG Task to get parent data
|
||||
Log::info("=== STEP 2: SCRAPING PBG TASK PARENT DATA ===");
|
||||
$import_datasource->update(['message' => 'Scraping PBG Task parent data...']);
|
||||
|
||||
$service_pbg_task->run_service();
|
||||
try{
|
||||
$service_tab_pbg_task->run_service();
|
||||
}catch(\Exception $e){
|
||||
$failed_uuid = $service_tab_pbg_task->getFailedUUID();
|
||||
throw $e;
|
||||
}
|
||||
Log::info("PBG Task parent data scraping completed");
|
||||
|
||||
// STEP 3: Get all PBG tasks for detail scraping
|
||||
$totalTasks = PbgTask::count();
|
||||
Log::info("=== STEP 3: SCRAPING PBG TASK DETAILS ===", [
|
||||
'total_tasks' => $totalTasks
|
||||
]);
|
||||
|
||||
$import_datasource->update([
|
||||
'message' => "Scraping details for {$totalTasks} PBG tasks..."
|
||||
]);
|
||||
|
||||
// Process tasks in chunks for memory efficiency
|
||||
$chunkSize = 100;
|
||||
$processedTasks = 0;
|
||||
|
||||
PbgTask::orderBy('id')->chunk($chunkSize, function ($pbg_tasks) use (
|
||||
$service_tab_pbg_task,
|
||||
&$processedTasks,
|
||||
$totalTasks,
|
||||
$import_datasource,
|
||||
&$failed_uuid
|
||||
) {
|
||||
foreach ($pbg_tasks as $pbg_task) {
|
||||
try {
|
||||
// Scrape all details for this task
|
||||
$this->processTaskDetails($service_tab_pbg_task, $pbg_task->uuid);
|
||||
|
||||
$processedTasks++;
|
||||
|
||||
// Update progress every 10 tasks
|
||||
if ($processedTasks % 10 === 0) {
|
||||
$progress = round(($processedTasks / $totalTasks) * 100, 2);
|
||||
Log::info("Progress update", [
|
||||
'processed' => $processedTasks,
|
||||
'total' => $totalTasks,
|
||||
'progress' => "{$progress}%"
|
||||
]);
|
||||
|
||||
$import_datasource->update([
|
||||
'message' => "Processing details: {$processedTasks}/{$totalTasks} ({$progress}%)"
|
||||
]);
|
||||
}
|
||||
|
||||
} catch (\Exception $e) {
|
||||
Log::warning("Failed to process task details", [
|
||||
'uuid' => $pbg_task->uuid,
|
||||
'error' => $e->getMessage()
|
||||
]);
|
||||
|
||||
// Store failed UUID but continue processing
|
||||
$failed_uuid = $pbg_task->uuid;
|
||||
|
||||
// Only stop if it's a critical error
|
||||
if ($this->isCriticalError($e)) {
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Log::info("Task details scraping completed", [
|
||||
'processed_tasks' => $processedTasks,
|
||||
'total_tasks' => $totalTasks
|
||||
]);
|
||||
|
||||
// STEP 4: Generate BigData Resume
|
||||
Log::info("=== STEP 4: GENERATING BIGDATA RESUME ===");
|
||||
$import_datasource->update(['message' => 'Generating BigData resume...']);
|
||||
|
||||
$data_setting_result = $service_google_sheet->get_big_resume_data();
|
||||
BigdataResume::generateResumeData($import_datasource->id, "simbg", $data_setting_result);
|
||||
|
||||
Log::info("BigData resume generated successfully");
|
||||
|
||||
// BigdataResume::generateResumeData($import_datasource->id, "all", $data_setting_result);
|
||||
BigdataResume::generateResumeData($import_datasource->id, now()->year, $data_setting_result);
|
||||
|
||||
// Update status to success
|
||||
// Update final status
|
||||
$import_datasource->update([
|
||||
'status' => 'success',
|
||||
'message' => 'Scraping completed successfully.',
|
||||
'finish_time' => now()
|
||||
'message' => "Scraping completed successfully. Processed {$processedTasks}/{$totalTasks} tasks.",
|
||||
'finish_time' => now(),
|
||||
'failed_uuid' => $failed_uuid // Store last failed UUID if any
|
||||
]);
|
||||
|
||||
Log::info("=== SCRAPING DATA JOB COMPLETED SUCCESSFULLY ===", [
|
||||
'import_datasource_id' => $import_datasource->id,
|
||||
'processed_tasks' => $processedTasks,
|
||||
'total_tasks' => $totalTasks,
|
||||
'has_failures' => !is_null($failed_uuid)
|
||||
]);
|
||||
|
||||
} catch (\Exception $e) {
|
||||
Log::error('Scraping failed: ' . $e->getMessage(), ['trace' => $e->getTraceAsString()]);
|
||||
Log::error('=== SCRAPING DATA JOB FAILED ===', [
|
||||
'error' => $e->getMessage(),
|
||||
'file' => $e->getFile(),
|
||||
'line' => $e->getLine(),
|
||||
'processed_tasks' => $processedTasks,
|
||||
'total_tasks' => $totalTasks,
|
||||
'failed_uuid' => $failed_uuid,
|
||||
'trace' => $e->getTraceAsString()
|
||||
]);
|
||||
|
||||
// Update status to failed
|
||||
if (isset($import_datasource)) {
|
||||
// Update ImportDatasource with failure info
|
||||
if ($import_datasource) {
|
||||
$import_datasource->update([
|
||||
'status' => 'failed',
|
||||
'message' => 'Terjadi kesalahan, Syncronize tidak selesai',
|
||||
'response_body' => 'Terjadi kesalahan, Syncronize tidak selesai',
|
||||
'message' => "Scraping failed: {$e->getMessage()}. Processed {$processedTasks}/{$totalTasks} tasks.",
|
||||
'response_body' => 'Scraping process interrupted due to error',
|
||||
'finish_time' => now(),
|
||||
'failed_uuid' => $failed_uuid,
|
||||
]);
|
||||
}
|
||||
|
||||
// Mark the job as failed
|
||||
// Don't retry this job
|
||||
$this->fail($e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process all detail endpoints for a single PBG task
|
||||
*/
|
||||
private function processTaskDetails(ServiceTabPbgTask $service, string $uuid): void
|
||||
{
|
||||
// Call all detail scraping methods for this task
|
||||
$service->scraping_task_details($uuid);
|
||||
$service->scraping_pbg_data_list($uuid);
|
||||
$service->scraping_task_retributions($uuid);
|
||||
$service->scraping_task_integrations($uuid);
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine if an error is critical enough to stop the entire process
|
||||
*/
|
||||
private function isCriticalError(\Exception $e): bool
|
||||
{
|
||||
$criticalMessages = [
|
||||
'authentication failed',
|
||||
'token expired',
|
||||
'database connection',
|
||||
'memory exhausted',
|
||||
'maximum execution time'
|
||||
];
|
||||
|
||||
$errorMessage = strtolower($e->getMessage());
|
||||
|
||||
foreach ($criticalMessages as $critical) {
|
||||
if (strpos($errorMessage, $critical) !== false) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,4 +54,86 @@ class PbgTask extends Model
|
||||
public function attachments(){
|
||||
return $this->hasMany(PbgTaskAttachment::class, 'pbg_task_id', 'id');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the data lists associated with this PBG task (One to Many)
|
||||
* One pbg_task can have many data lists
|
||||
*/
|
||||
public function dataLists()
|
||||
{
|
||||
return $this->hasMany(PbgTaskDetailDataList::class, 'pbg_task_uuid', 'uuid');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get only data lists with files
|
||||
*/
|
||||
public function dataListsWithFiles()
|
||||
{
|
||||
return $this->hasMany(PbgTaskDetailDataList::class, 'pbg_task_uuid', 'uuid')
|
||||
->whereNotNull('file')
|
||||
->where('file', '!=', '');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get data lists by status
|
||||
*/
|
||||
public function dataListsByStatus($status)
|
||||
{
|
||||
return $this->hasMany(PbgTaskDetailDataList::class, 'pbg_task_uuid', 'uuid')
|
||||
->where('status', $status);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get data lists by data type
|
||||
*/
|
||||
public function dataListsByType($dataType)
|
||||
{
|
||||
return $this->hasMany(PbgTaskDetailDataList::class, 'pbg_task_uuid', 'uuid')
|
||||
->where('data_type', $dataType);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create or update data lists from API response
|
||||
*/
|
||||
public function syncDataLists(array $dataLists): void
|
||||
{
|
||||
foreach ($dataLists as $listData) {
|
||||
PbgTaskDetailDataList::updateOrCreate(
|
||||
['uid' => $listData['uid']],
|
||||
[
|
||||
'name' => $listData['name'] ?? null,
|
||||
'description' => $listData['description'] ?? null,
|
||||
'status' => $listData['status'] ?? null,
|
||||
'status_name' => $listData['status_name'] ?? null,
|
||||
'data_type' => $listData['data_type'] ?? null,
|
||||
'data_type_name' => $listData['data_type_name'] ?? null,
|
||||
'file' => $listData['file'] ?? null,
|
||||
'note' => $listData['note'] ?? null,
|
||||
'pbg_task_uuid' => $this->uuid,
|
||||
]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get data lists count by status
|
||||
*/
|
||||
public function getDataListsCountByStatusAttribute()
|
||||
{
|
||||
return $this->dataLists()
|
||||
->selectRaw('status, COUNT(*) as count')
|
||||
->groupBy('status')
|
||||
->pluck('count', 'status');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get data lists count by data type
|
||||
*/
|
||||
public function getDataListsCountByTypeAttribute()
|
||||
{
|
||||
return $this->dataLists()
|
||||
->selectRaw('data_type, COUNT(*) as count')
|
||||
->groupBy('data_type')
|
||||
->pluck('count', 'data_type');
|
||||
}
|
||||
}
|
||||
|
||||
@@ -141,6 +141,8 @@ class PbgTaskDetail extends Model
|
||||
return $this->belongsTo(PbgTask::class, 'pbg_task_uid', 'uuid');
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Create or update PbgTaskDetail from API response
|
||||
*/
|
||||
@@ -249,4 +251,6 @@ class PbgTaskDetail extends Model
|
||||
$detailData
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
161
app/Models/PbgTaskDetailDataList.php
Normal file
161
app/Models/PbgTaskDetailDataList.php
Normal file
@@ -0,0 +1,161 @@
|
||||
<?php
|
||||
|
||||
namespace App\Models;
|
||||
|
||||
use Illuminate\Database\Eloquent\Factories\HasFactory;
|
||||
use Illuminate\Database\Eloquent\Model;
|
||||
|
||||
class PbgTaskDetailDataList extends Model
|
||||
{
|
||||
use HasFactory;
|
||||
|
||||
protected $table = 'pbg_task_detail_data_lists';
|
||||
|
||||
protected $fillable = [
|
||||
'uid',
|
||||
'name',
|
||||
'description',
|
||||
'status',
|
||||
'status_name',
|
||||
'data_type',
|
||||
'data_type_name',
|
||||
'file',
|
||||
'note',
|
||||
'pbg_task_uuid',
|
||||
];
|
||||
|
||||
protected $casts = [
|
||||
'status' => 'integer',
|
||||
'data_type' => 'integer',
|
||||
'created_at' => 'datetime',
|
||||
'updated_at' => 'datetime',
|
||||
];
|
||||
|
||||
/**
|
||||
* Relationship to PbgTask (Many to One)
|
||||
* Many data lists belong to one pbg_task
|
||||
*/
|
||||
public function pbgTask()
|
||||
{
|
||||
return $this->belongsTo(PbgTask::class, 'pbg_task_uuid', 'uuid');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the full file path
|
||||
*/
|
||||
public function getFilePathAttribute()
|
||||
{
|
||||
return $this->file ? storage_path('app/public/' . $this->file) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the file URL
|
||||
*/
|
||||
public function getFileUrlAttribute()
|
||||
{
|
||||
return $this->file ? asset('storage/' . $this->file) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if file exists
|
||||
*/
|
||||
public function hasFile()
|
||||
{
|
||||
return !empty($this->file) && file_exists($this->getFilePathAttribute());
|
||||
}
|
||||
|
||||
/**
|
||||
* Get status badge color based on status
|
||||
*/
|
||||
public function getStatusBadgeAttribute()
|
||||
{
|
||||
return match($this->status) {
|
||||
1 => 'success', // Sesuai
|
||||
0 => 'danger', // Tidak Sesuai
|
||||
default => 'secondary'
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Scope: Filter by status
|
||||
*/
|
||||
public function scopeByStatus($query, $status)
|
||||
{
|
||||
return $query->where('status', $status);
|
||||
}
|
||||
|
||||
/**
|
||||
* Scope: Filter by data type
|
||||
*/
|
||||
public function scopeByDataType($query, $dataType)
|
||||
{
|
||||
return $query->where('data_type', $dataType);
|
||||
}
|
||||
|
||||
/**
|
||||
* Scope: With files only
|
||||
*/
|
||||
public function scopeWithFiles($query)
|
||||
{
|
||||
return $query->whereNotNull('file')->where('file', '!=', '');
|
||||
}
|
||||
|
||||
/**
|
||||
* Scope: Search by name or description
|
||||
*/
|
||||
public function scopeSearch($query, $search)
|
||||
{
|
||||
return $query->where(function ($q) use ($search) {
|
||||
$q->where('name', 'LIKE', "%{$search}%")
|
||||
->orWhere('description', 'LIKE', "%{$search}%")
|
||||
->orWhere('status_name', 'LIKE', "%{$search}%")
|
||||
->orWhere('data_type_name', 'LIKE', "%{$search}%");
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get file extension from file path
|
||||
*/
|
||||
public function getFileExtensionAttribute()
|
||||
{
|
||||
if (!$this->file) {
|
||||
return null;
|
||||
}
|
||||
return strtoupper(pathinfo($this->file, PATHINFO_EXTENSION));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get filename from file path
|
||||
*/
|
||||
public function getFileNameAttribute()
|
||||
{
|
||||
if (!$this->file) {
|
||||
return null;
|
||||
}
|
||||
return basename($this->file);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get formatted created date
|
||||
*/
|
||||
public function getFormattedCreatedAtAttribute()
|
||||
{
|
||||
return $this->created_at ? $this->created_at->format('d M Y, H:i') : '-';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get truncated description
|
||||
*/
|
||||
public function getTruncatedDescriptionAttribute()
|
||||
{
|
||||
return $this->description ? \Str::limit($this->description, 80) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get truncated note
|
||||
*/
|
||||
public function getTruncatedNoteAttribute()
|
||||
{
|
||||
return $this->note ? \Str::limit($this->note, 100) : null;
|
||||
}
|
||||
}
|
||||
@@ -106,7 +106,7 @@ class ServicePbgTask
|
||||
};
|
||||
|
||||
do {
|
||||
$url = "{$this->simbg_host}/api/pbg/v1/list/?page={$currentPage}&size={$this->fetch_per_page}&sort=ASC&date&search&status&slf_status&type&sort_by=created_at&application_type=1&start_date&end_date";
|
||||
$url = "{$this->simbg_host}/api/pbg/v1/list/?page={$currentPage}&size={$this->fetch_per_page}&sort=ASC&date&search&status&slf_status&type=task&sort_by=created_at&application_type=1&start_date&end_date";
|
||||
|
||||
$fetch_data = $fetchData($url);
|
||||
if (!$fetch_data) {
|
||||
|
||||
@@ -5,6 +5,7 @@ namespace App\Services;
|
||||
use App\Models\GlobalSetting;
|
||||
use App\Models\PbgTask;
|
||||
use App\Models\PbgTaskDetail;
|
||||
use App\Models\PbgTaskDetailDataList;
|
||||
use App\Models\PbgTaskIndexIntegrations;
|
||||
use App\Models\PbgTaskPrasarana;
|
||||
use App\Models\PbgTaskRetributions;
|
||||
@@ -36,44 +37,119 @@ class ServiceTabPbgTask
|
||||
$this->user_refresh_token = $auth_data['refresh'];
|
||||
}
|
||||
|
||||
public function run_service($retry_uuid = null)
|
||||
public function run_service($retry_uuid = null, $chunk_size = 50)
|
||||
{
|
||||
try {
|
||||
$pbg_tasks = PbgTask::orderBy('id')->get();
|
||||
$start = false;
|
||||
$query = PbgTask::orderBy('id');
|
||||
|
||||
// If retry_uuid is provided, start from that UUID
|
||||
if ($retry_uuid) {
|
||||
$retryTask = PbgTask::where('uuid', $retry_uuid)->first();
|
||||
if ($retryTask) {
|
||||
$query->where('id', '>=', $retryTask->id);
|
||||
Log::info("Resuming sync from UUID: {$retry_uuid} (ID: {$retryTask->id})");
|
||||
}
|
||||
}
|
||||
|
||||
foreach ($pbg_tasks as $pbg_task) {
|
||||
if($retry_uuid){
|
||||
if($pbg_task->uuid === $retry_uuid){
|
||||
$start = true;
|
||||
}
|
||||
$totalTasks = $query->count();
|
||||
$processedCount = 0;
|
||||
|
||||
Log::info("Starting sync for {$totalTasks} PBG Tasks with chunk size: {$chunk_size}");
|
||||
|
||||
if(!$start){
|
||||
// Process in chunks to reduce memory usage
|
||||
$query->chunk($chunk_size, function ($pbg_tasks) use (&$processedCount, $totalTasks) {
|
||||
$chunkStartTime = now();
|
||||
|
||||
foreach ($pbg_tasks as $pbg_task) {
|
||||
try {
|
||||
$this->current_uuid = $pbg_task->uuid;
|
||||
$taskStartTime = now();
|
||||
|
||||
// Process all endpoints for this task
|
||||
$this->processTaskEndpoints($pbg_task->uuid);
|
||||
|
||||
$processedCount++;
|
||||
$taskTime = now()->diffInSeconds($taskStartTime);
|
||||
|
||||
// Log progress every 10 tasks
|
||||
if ($processedCount % 10 === 0) {
|
||||
$progress = round(($processedCount / $totalTasks) * 100, 2);
|
||||
Log::info("Progress: {$processedCount}/{$totalTasks} ({$progress}%) - Last task took {$taskTime}s");
|
||||
}
|
||||
|
||||
} catch (\Exception $e) {
|
||||
Log::error("Failed on UUID: {$this->current_uuid}, Error: " . $e->getMessage());
|
||||
|
||||
// Check if this is a critical error that should stop the process
|
||||
if ($this->isCriticalError($e)) {
|
||||
throw $e;
|
||||
}
|
||||
|
||||
// For non-critical errors, log and continue
|
||||
Log::warning("Skipping UUID {$this->current_uuid} due to non-critical error");
|
||||
continue;
|
||||
}
|
||||
}
|
||||
try{
|
||||
$this->current_uuid = $pbg_task->uuid;
|
||||
$this->scraping_task_details($pbg_task->uuid);
|
||||
// $this->scraping_task_assignments($pbg_task->uuid);
|
||||
$this->scraping_task_retributions($pbg_task->uuid);
|
||||
$this->scraping_task_integrations($pbg_task->uuid);
|
||||
}catch(\Exception $e){
|
||||
Log::error("Failed on UUID: {$this->current_uuid}, Error: " . $e->getMessage());
|
||||
throw $e;
|
||||
|
||||
$chunkTime = now()->diffInSeconds($chunkStartTime);
|
||||
Log::info("Processed chunk of {$pbg_tasks->count()} tasks in {$chunkTime} seconds");
|
||||
|
||||
// Small delay between chunks to prevent API rate limiting
|
||||
if ($pbg_tasks->count() === $chunk_size) {
|
||||
sleep(1);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
Log::info("Successfully completed sync for {$processedCount} PBG Tasks");
|
||||
|
||||
} catch (\Exception $e) {
|
||||
Log::error("Failed to syncronize: " . $e->getMessage());
|
||||
Log::error("Failed to synchronize: " . $e->getMessage());
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process all endpoints for a single task
|
||||
*/
|
||||
private function processTaskEndpoints(string $uuid): void
|
||||
{
|
||||
$this->scraping_task_details($uuid);
|
||||
$this->scraping_pbg_data_list($uuid);
|
||||
// $this->scraping_task_assignments($uuid);
|
||||
$this->scraping_task_retributions($uuid);
|
||||
$this->scraping_task_integrations($uuid);
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine if an error is critical and should stop the process
|
||||
*/
|
||||
private function isCriticalError(\Exception $e): bool
|
||||
{
|
||||
$message = $e->getMessage();
|
||||
|
||||
// Critical authentication errors
|
||||
if (strpos($message, 'Token refresh and login failed') !== false) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Critical system errors
|
||||
if (strpos($message, 'Connection refused') !== false) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Database connection errors
|
||||
if (strpos($message, 'database') !== false && strpos($message, 'connection') !== false) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public function getFailedUUID(){
|
||||
return $this->current_uuid;
|
||||
}
|
||||
|
||||
private function scraping_task_details($uuid)
|
||||
public function scraping_task_details($uuid)
|
||||
{
|
||||
$url = "{$this->simbg_host}/api/pbg/v1/detail/{$uuid}/";
|
||||
$options = [
|
||||
@@ -144,7 +220,7 @@ class ServiceTabPbgTask
|
||||
throw new \Exception("Failed to fetch task details for UUID {$uuid} after retries.");
|
||||
}
|
||||
|
||||
private function scraping_task_assignments($uuid)
|
||||
public function scraping_task_assignments($uuid)
|
||||
{
|
||||
$url = "{$this->simbg_host}/api/pbg/v1/list-tim-penilai/{$uuid}/?page=1&size=10";
|
||||
$options = [
|
||||
@@ -237,7 +313,173 @@ class ServiceTabPbgTask
|
||||
throw new \Exception("Failed to fetch task assignments for UUID {$uuid} after retries.");
|
||||
}
|
||||
|
||||
private function scraping_task_retributions($uuid)
|
||||
public function scraping_pbg_data_list($uuid){
|
||||
$url = "{$this->simbg_host}/api/pbg/v1/detail/{$uuid}/list-data/?sort=DESC";
|
||||
$options = [
|
||||
'headers' => [
|
||||
'Authorization' => "Bearer {$this->user_token}",
|
||||
'Content-Type' => 'application/json'
|
||||
]
|
||||
];
|
||||
|
||||
$maxRetries = 3;
|
||||
$initialDelay = 1;
|
||||
$retriedAfter401 = false;
|
||||
|
||||
for ($retryCount = 0; $retryCount < $maxRetries; $retryCount++) {
|
||||
try{
|
||||
$response = $this->client->get($url, $options);
|
||||
$responseData = json_decode($response->getBody()->getContents(), true, 512, JSON_THROW_ON_ERROR);
|
||||
|
||||
if (empty($responseData['data']) || !is_array($responseData['data'])) {
|
||||
Log::info("No data list found for UUID: {$uuid}");
|
||||
return true;
|
||||
}
|
||||
|
||||
$data = $responseData['data'];
|
||||
|
||||
Log::info("Processing data list for UUID: {$uuid}, found " . count($data) . " items");
|
||||
|
||||
// Process each data list item and save to database
|
||||
$this->processDataListItems($data, $uuid);
|
||||
|
||||
return $responseData;
|
||||
} catch (\GuzzleHttp\Exception\ClientException $e) {
|
||||
if ($e->getCode() === 401 && !$retriedAfter401) {
|
||||
Log::warning("401 Unauthorized - Refreshing token and retrying...");
|
||||
try{
|
||||
$this->refreshToken();
|
||||
$options['headers']['Authorization'] = "Bearer {$this->user_token}";
|
||||
$retriedAfter401 = true;
|
||||
continue;
|
||||
}catch(\Exception $refreshError){
|
||||
Log::error("Token refresh and login failed: " . $refreshError->getMessage());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
} catch (\GuzzleHttp\Exception\ServerException | \GuzzleHttp\Exception\ConnectException $e) {
|
||||
if ($e->getCode() === 502) {
|
||||
Log::warning("502 Bad Gateway - Retrying in {$initialDelay} seconds...");
|
||||
} else {
|
||||
Log::error("Network error ({$e->getCode()}) - Retrying in {$initialDelay} seconds...");
|
||||
}
|
||||
|
||||
sleep($initialDelay);
|
||||
$initialDelay *= 2;
|
||||
} catch (\GuzzleHttp\Exception\RequestException $e) {
|
||||
Log::error("Request error ({$e->getCode()}): " . $e->getMessage());
|
||||
return false;
|
||||
} catch (\JsonException $e) {
|
||||
Log::error("JSON decoding error: " . $e->getMessage());
|
||||
return false;
|
||||
} catch (\Throwable $e) {
|
||||
Log::critical("Unhandled error: " . $e->getMessage(), ['trace' => $e->getTraceAsString()]);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
Log::error("Failed to fetch task data list for UUID {$uuid} after {$maxRetries} retries.");
|
||||
throw new \Exception("Failed to fetch task data list for UUID {$uuid} after retries.");
|
||||
}
|
||||
|
||||
/**
|
||||
* Process and save data list items to database (Optimized with bulk operations)
|
||||
*/
|
||||
private function processDataListItems(array $dataListItems, string $pbgTaskUuid): void
|
||||
{
|
||||
try {
|
||||
if (empty($dataListItems)) {
|
||||
return;
|
||||
}
|
||||
|
||||
$batchData = [];
|
||||
$validItems = 0;
|
||||
|
||||
foreach ($dataListItems as $item) {
|
||||
// Validate required fields
|
||||
if (empty($item['uid'])) {
|
||||
Log::warning("Skipping data list item with missing UID for PBG Task: {$pbgTaskUuid}");
|
||||
continue;
|
||||
}
|
||||
|
||||
// Parse created_at if exists
|
||||
$createdAt = null;
|
||||
if (!empty($item['created_at'])) {
|
||||
try {
|
||||
$createdAt = Carbon::parse($item['created_at'])->format('Y-m-d H:i:s');
|
||||
} catch (\Exception $e) {
|
||||
Log::warning("Invalid created_at format for data list UID: {$item['uid']}, Error: " . $e->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
$batchData[] = [
|
||||
'uid' => $item['uid'],
|
||||
'name' => $item['name'] ?? null,
|
||||
'description' => $item['description'] ?? null,
|
||||
'status' => $item['status'] ?? null,
|
||||
'status_name' => $item['status_name'] ?? null,
|
||||
'data_type' => $item['data_type'] ?? null,
|
||||
'data_type_name' => $item['data_type_name'] ?? null,
|
||||
'file' => $item['file'] ?? null,
|
||||
'note' => $item['note'] ?? null,
|
||||
'pbg_task_uuid' => $pbgTaskUuid,
|
||||
'created_at' => $createdAt ?: now(),
|
||||
'updated_at' => now(),
|
||||
];
|
||||
|
||||
$validItems++;
|
||||
}
|
||||
|
||||
if (!empty($batchData)) {
|
||||
// Use upsert for bulk insert/update operations
|
||||
PbgTaskDetailDataList::upsert(
|
||||
$batchData,
|
||||
['uid'], // Unique columns
|
||||
[
|
||||
'name', 'description', 'status', 'status_name',
|
||||
'data_type', 'data_type_name', 'file', 'note',
|
||||
'pbg_task_uuid', 'updated_at'
|
||||
] // Columns to update
|
||||
);
|
||||
|
||||
Log::info("Successfully bulk processed {$validItems} data list items for PBG Task: {$pbgTaskUuid}");
|
||||
}
|
||||
|
||||
} catch (\Exception $e) {
|
||||
Log::error("Error bulk processing data list items for PBG Task {$pbgTaskUuid}: " . $e->getMessage());
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Alternative method using PbgTask model's syncDataLists for cleaner code
|
||||
*/
|
||||
private function processDataListItemsWithModel(array $dataListItems, string $pbgTaskUuid): void
|
||||
{
|
||||
try {
|
||||
// Find the PbgTask
|
||||
$pbgTask = PbgTask::where('uuid', $pbgTaskUuid)->first();
|
||||
|
||||
if (!$pbgTask) {
|
||||
Log::error("PBG Task not found with UUID: {$pbgTaskUuid}");
|
||||
return;
|
||||
}
|
||||
|
||||
// Use the model's syncDataLists method
|
||||
$pbgTask->syncDataLists($dataListItems);
|
||||
|
||||
$processedCount = count($dataListItems);
|
||||
Log::info("Successfully synced {$processedCount} data list items for PBG Task: {$pbgTaskUuid} using model method");
|
||||
|
||||
} catch (\Exception $e) {
|
||||
Log::error("Error syncing data list items for PBG Task {$pbgTaskUuid}: " . $e->getMessage());
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
public function scraping_task_retributions($uuid)
|
||||
{
|
||||
$url = "{$this->simbg_host}/api/pbg/v1/detail/" . $uuid . "/retribution/submit/";
|
||||
$options = [
|
||||
@@ -354,7 +596,7 @@ class ServiceTabPbgTask
|
||||
throw new \Exception("Failed to fetch task retributions for UUID {$uuid} after retries.");
|
||||
}
|
||||
|
||||
private function scraping_task_integrations($uuid){
|
||||
public function scraping_task_integrations($uuid){
|
||||
$url = "{$this->simbg_host}/api/pbg/v1/detail/" . $uuid . "/retribution/indeks-terintegrasi/";
|
||||
$options = [
|
||||
'headers' => [
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
<?php
|
||||
|
||||
use Illuminate\Database\Migrations\Migration;
|
||||
use Illuminate\Database\Schema\Blueprint;
|
||||
use Illuminate\Support\Facades\Schema;
|
||||
|
||||
return new class extends Migration
|
||||
{
|
||||
/**
|
||||
* Run the migrations.
|
||||
*/
|
||||
public function up(): void
|
||||
{
|
||||
Schema::create('pbg_task_detail_data_lists', function (Blueprint $table) {
|
||||
$table->id();
|
||||
$table->string('uid')->unique(); // UID from response
|
||||
$table->string('name'); // Nama data
|
||||
$table->text('description')->nullable(); // Deskripsi (bisa null)
|
||||
$table->integer('status')->nullable(); // Status (1 = Sesuai, etc)
|
||||
$table->string('status_name')->nullable(); // Nama status
|
||||
$table->integer('data_type')->nullable(); // Tipe data (1 = Data Teknis Tanah, etc)
|
||||
$table->string('data_type_name')->nullable(); // Nama tipe data
|
||||
$table->text('file')->nullable(); // Path file
|
||||
$table->text('note')->nullable(); // Catatan
|
||||
|
||||
// Foreign key ke pbg_task (1 to many relationship)
|
||||
$table->string('pbg_task_uuid')->nullable();
|
||||
$table->foreign('pbg_task_uuid')->references('uuid')->on('pbg_task')->onDelete('cascade');
|
||||
|
||||
// Indexes for better performance
|
||||
$table->index('uid');
|
||||
$table->index('pbg_task_uuid');
|
||||
$table->index('status');
|
||||
$table->index('data_type');
|
||||
|
||||
$table->timestamps();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Reverse the migrations.
|
||||
*/
|
||||
public function down(): void
|
||||
{
|
||||
Schema::dropIfExists('pbg_task_detail_data_lists');
|
||||
}
|
||||
};
|
||||
214
resources/scss/pages/pbg-task/show.scss
Normal file
214
resources/scss/pages/pbg-task/show.scss
Normal file
@@ -0,0 +1,214 @@
|
||||
// PBG Task Show Page Styles
|
||||
// Custom styles for data lists display (List Layout)
|
||||
|
||||
.data-list-section {
|
||||
.section-header {
|
||||
border-bottom: 2px solid #f8f9fa;
|
||||
padding-bottom: 0.75rem;
|
||||
|
||||
.section-icon {
|
||||
font-size: 1.2rem;
|
||||
color: #5d87ff;
|
||||
}
|
||||
|
||||
.section-title {
|
||||
font-size: 1.1rem;
|
||||
font-weight: 600;
|
||||
color: #2c3e50;
|
||||
}
|
||||
|
||||
.section-count {
|
||||
font-size: 0.85rem;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.data-list-container {
|
||||
.data-list-item {
|
||||
border: 1px solid #e9ecef;
|
||||
border-radius: 6px;
|
||||
margin-bottom: 0.75rem;
|
||||
padding: 1rem;
|
||||
background: #ffffff;
|
||||
transition: all 0.2s ease;
|
||||
position: relative;
|
||||
|
||||
&:hover {
|
||||
border-color: #5d87ff;
|
||||
box-shadow: 0 2px 8px rgba(93, 135, 255, 0.1);
|
||||
transform: translateX(2px);
|
||||
}
|
||||
|
||||
&::before {
|
||||
content: "";
|
||||
position: absolute;
|
||||
left: 0;
|
||||
top: 0;
|
||||
bottom: 0;
|
||||
width: 3px;
|
||||
background: #5d87ff;
|
||||
border-radius: 3px 0 0 3px;
|
||||
opacity: 0;
|
||||
transition: opacity 0.2s ease;
|
||||
}
|
||||
|
||||
&:hover::before {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
.list-item-header {
|
||||
.item-title {
|
||||
font-size: 1rem;
|
||||
font-weight: 600;
|
||||
color: #2c3e50;
|
||||
line-height: 1.4;
|
||||
|
||||
.item-number {
|
||||
color: #5d87ff;
|
||||
font-weight: 700;
|
||||
margin-right: 0.5rem;
|
||||
}
|
||||
}
|
||||
|
||||
.item-description {
|
||||
font-size: 0.9rem;
|
||||
line-height: 1.4;
|
||||
color: #6c757d;
|
||||
margin-bottom: 0.5rem;
|
||||
}
|
||||
|
||||
.item-status {
|
||||
.badge {
|
||||
font-size: 0.75rem;
|
||||
padding: 0.25rem 0.6rem;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.list-item-meta {
|
||||
margin-top: 0.75rem;
|
||||
padding-top: 0.75rem;
|
||||
border-top: 1px solid #f1f3f4;
|
||||
|
||||
small {
|
||||
font-size: 0.8rem;
|
||||
|
||||
i {
|
||||
opacity: 0.7;
|
||||
}
|
||||
|
||||
.text-dark {
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.badge {
|
||||
font-size: 0.7rem;
|
||||
padding: 0.1rem 0.4rem;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Empty state for individual sections
|
||||
.no-items {
|
||||
text-align: center;
|
||||
padding: 2rem;
|
||||
color: #6c757d;
|
||||
font-style: italic;
|
||||
border: 2px dashed #dee2e6;
|
||||
border-radius: 6px;
|
||||
|
||||
i {
|
||||
font-size: 2rem;
|
||||
margin-bottom: 0.5rem;
|
||||
opacity: 0.5;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Empty state styles
|
||||
.empty-state {
|
||||
text-align: center;
|
||||
padding: 3rem 1rem;
|
||||
|
||||
.empty-icon {
|
||||
font-size: 3rem;
|
||||
color: #6c757d;
|
||||
margin-bottom: 1rem;
|
||||
opacity: 0.6;
|
||||
}
|
||||
|
||||
.empty-title {
|
||||
font-size: 1.25rem;
|
||||
color: #6c757d;
|
||||
margin-bottom: 0.5rem;
|
||||
}
|
||||
|
||||
.empty-text {
|
||||
color: #6c757d;
|
||||
opacity: 0.8;
|
||||
}
|
||||
}
|
||||
|
||||
// Badge variations
|
||||
.badge {
|
||||
&.bg-success {
|
||||
background-color: #28a745 !important;
|
||||
}
|
||||
|
||||
&.bg-danger {
|
||||
background-color: #dc3545 !important;
|
||||
}
|
||||
|
||||
&.bg-info {
|
||||
background-color: #17a2b8 !important;
|
||||
}
|
||||
|
||||
&.bg-light {
|
||||
background-color: #f8f9fa !important;
|
||||
border: 1px solid #dee2e6;
|
||||
}
|
||||
}
|
||||
|
||||
// Tab content spacing
|
||||
#pbgTaskDataLists {
|
||||
.section-divider {
|
||||
border-bottom: 1px solid #e9ecef;
|
||||
margin-bottom: 2rem;
|
||||
padding-bottom: 1rem;
|
||||
|
||||
&:last-child {
|
||||
border-bottom: none;
|
||||
margin-bottom: 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Responsive adjustments
|
||||
@media (max-width: 768px) {
|
||||
.data-list-container {
|
||||
.data-list-item {
|
||||
padding: 0.75rem;
|
||||
|
||||
.list-item-header {
|
||||
.item-title {
|
||||
font-size: 0.95rem;
|
||||
}
|
||||
}
|
||||
|
||||
.list-item-meta {
|
||||
small {
|
||||
font-size: 0.75rem;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.data-list-section {
|
||||
.section-header {
|
||||
.section-title {
|
||||
font-size: 1rem;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -38,7 +38,7 @@
|
||||
</div>
|
||||
|
||||
<!-- Notification -->
|
||||
<div class="dropdown topbar-item">
|
||||
{{-- <div class="dropdown topbar-item">
|
||||
<button type="button" class="topbar-button position-relative"
|
||||
id="page-header-notifications-dropdown" data-bs-toggle="dropdown" aria-haspopup="true"
|
||||
aria-expanded="false">
|
||||
@@ -147,7 +147,7 @@
|
||||
class="bx bx-right-arrow-alt ms-1"></i></a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div> --}}
|
||||
|
||||
<!-- User -->
|
||||
<div class="dropdown topbar-item">
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
@section('css')
|
||||
@vite(['node_modules/gridjs/dist/theme/mermaid.min.css'])
|
||||
@vite(['resources/scss/pages/pbg-task/show.scss'])
|
||||
@endsection
|
||||
|
||||
@section('content')
|
||||
@@ -127,6 +128,11 @@
|
||||
<span class="d-none d-sm-block">Data Bangunan</span>
|
||||
</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
<a href="#pbgTaskDataLists" data-bs-toggle="tab" aria-expanded="false" class="nav-link">
|
||||
<span class="d-none d-sm-block">Data Lists</span>
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
@@ -398,6 +404,99 @@
|
||||
</div>
|
||||
@endif
|
||||
</div>
|
||||
<div class="tab-pane" id="pbgTaskDataLists">
|
||||
@if($dataListsByType && $dataListsByType->count() > 0)
|
||||
@foreach($dataListsByType as $dataType => $dataLists)
|
||||
@php
|
||||
$dataTypeName = $dataLists->first()->data_type_name ?? "Data Type {$dataType}";
|
||||
@endphp
|
||||
<div class="data-list-section mb-4">
|
||||
<div class="section-header mb-3">
|
||||
<div class="d-flex align-items-center justify-content-between">
|
||||
<div class="d-flex align-items-center">
|
||||
<i class="fas fa-folder-open section-icon me-2"></i>
|
||||
<h5 class="section-title mb-0">{{ $dataTypeName }}</h5>
|
||||
</div>
|
||||
<span class="badge bg-info section-count">{{ $dataLists->count() }} items</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="data-list-container">
|
||||
@foreach($dataLists as $index => $dataList)
|
||||
<div class="data-list-item">
|
||||
<div class="list-item-header">
|
||||
<div class="d-flex align-items-start justify-content-between">
|
||||
<div class="item-info flex-grow-1">
|
||||
<h6 class="item-title mb-1">
|
||||
<span class="item-number">{{ $index + 1 }}.</span>
|
||||
{{ $dataList->name }}
|
||||
</h6>
|
||||
@if($dataList->description)
|
||||
<p class="item-description text-muted small mb-1">
|
||||
{{ $dataList->description }}
|
||||
</p>
|
||||
@endif
|
||||
</div>
|
||||
@if($dataList->status !== null)
|
||||
<div class="item-status ms-3">
|
||||
@if($dataList->status == 1)
|
||||
<span class="badge bg-success">{{ $dataList->status_name ?? 'Sesuai' }}</span>
|
||||
@elseif($dataList->status == 0)
|
||||
<span class="badge bg-danger">{{ $dataList->status_name ?? 'Tidak Sesuai' }}</span>
|
||||
@else
|
||||
<span class="badge bg-secondary">{{ $dataList->status_name ?? 'Unknown' }}</span>
|
||||
@endif
|
||||
</div>
|
||||
@endif
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="list-item-meta">
|
||||
<div class="row">
|
||||
@if($dataList->file)
|
||||
<div class="col-md-6">
|
||||
<small class="text-muted">
|
||||
<i class="fas fa-paperclip me-1"></i>
|
||||
File:
|
||||
<span class="text-dark">{{ $dataList->file_name }}</span>
|
||||
@if($dataList->file_extension)
|
||||
<span class="badge bg-light text-dark ms-1">{{ $dataList->file_extension }}</span>
|
||||
@endif
|
||||
</small>
|
||||
</div>
|
||||
@endif
|
||||
<div class="col-md-6">
|
||||
<small class="text-muted">
|
||||
<i class="fas fa-clock me-1"></i>
|
||||
{{ $dataList->formatted_created_at }}
|
||||
</small>
|
||||
</div>
|
||||
</div>
|
||||
@if($dataList->note)
|
||||
<div class="row mt-1">
|
||||
<div class="col-12">
|
||||
<small class="text-muted">
|
||||
<i class="fas fa-sticky-note me-1"></i>
|
||||
Catatan:
|
||||
<span class="text-dark">{{ $dataList->note }}</span>
|
||||
</small>
|
||||
</div>
|
||||
</div>
|
||||
@endif
|
||||
</div>
|
||||
</div>
|
||||
@endforeach
|
||||
</div>
|
||||
</div>
|
||||
@endforeach
|
||||
@else
|
||||
<div class="empty-state">
|
||||
<i class="fas fa-folder-open empty-icon"></i>
|
||||
<h5 class="empty-title">No Data Lists Available</h5>
|
||||
<p class="empty-text">There are no data lists associated with this PBG task.</p>
|
||||
</div>
|
||||
@endif
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
27
supervisor-scraping-queue.conf
Normal file
27
supervisor-scraping-queue.conf
Normal file
@@ -0,0 +1,27 @@
|
||||
# Supervisor Configuration for Scraping Queue Worker
|
||||
# Handles scraping operations with extended timeouts
|
||||
# File: /etc/supervisor/conf.d/sibedas-scraping.conf
|
||||
|
||||
# SCRAPING Queue Worker - For scraping operations (manual start/stop)
|
||||
[program:sibedas-queue-scraping]
|
||||
process_name=%(program_name)s
|
||||
command=php /var/www/artisan queue:work database --queue=scraping --sleep=1 --tries=1 --max-time=0 --memory=1024 --timeout=0
|
||||
directory=/var/www
|
||||
autostart=false
|
||||
autorestart=true
|
||||
startretries=3
|
||||
user=www-data
|
||||
numprocs=1
|
||||
redirect_stderr=true
|
||||
stdout_logfile=/var/log/supervisor/sibedas-queue-scraping.log
|
||||
stdout_logfile_maxbytes=500MB
|
||||
stdout_logfile_backups=3
|
||||
stopwaitsecs=120
|
||||
stopsignal=TERM
|
||||
killasgroup=true
|
||||
stopasgroup=true
|
||||
|
||||
# Group configuration for scraping services
|
||||
[group:sibedas-scraping]
|
||||
programs=sibedas-queue-scraping
|
||||
priority=998
|
||||
40
supervisor-upload-queue.conf
Normal file
40
supervisor-upload-queue.conf
Normal file
@@ -0,0 +1,40 @@
|
||||
# Supervisor Configuration for Upload Queue Worker
|
||||
# Handles file uploads and general tasks
|
||||
# File: /etc/supervisor/conf.d/sibedas-upload.conf
|
||||
|
||||
# Laravel Scheduler - Runs automated tasks
|
||||
[program:sibedas-scheduler]
|
||||
process_name=%(program_name)s
|
||||
command=php /var/www/artisan schedule:work
|
||||
directory=/var/www
|
||||
autostart=true
|
||||
autorestart=true
|
||||
startretries=3
|
||||
user=www-data
|
||||
numprocs=1
|
||||
redirect_stderr=true
|
||||
stdout_logfile=/var/log/supervisor/sibedas-scheduler.log
|
||||
stdout_logfile_maxbytes=50MB
|
||||
stdout_logfile_backups=5
|
||||
stopwaitsecs=10
|
||||
|
||||
# UPLOAD Queue Worker - For file uploads and general tasks (24/7)
|
||||
[program:sibedas-queue-upload]
|
||||
process_name=%(program_name)s
|
||||
command=php /var/www/artisan queue:work database --queue=upload --sleep=2 --tries=1 --max-time=3600 --memory=256 --timeout=3600
|
||||
directory=/var/www
|
||||
autostart=true
|
||||
autorestart=true
|
||||
startretries=3
|
||||
user=www-data
|
||||
numprocs=1
|
||||
redirect_stderr=true
|
||||
stdout_logfile=/var/log/supervisor/sibedas-queue-upload.log
|
||||
stdout_logfile_maxbytes=100MB
|
||||
stdout_logfile_backups=5
|
||||
stopwaitsecs=60
|
||||
|
||||
# Group configuration for upload services
|
||||
[group:sibedas-upload]
|
||||
programs=sibedas-scheduler,sibedas-queue-upload
|
||||
priority=999
|
||||
@@ -53,6 +53,7 @@ export default defineConfig(({ mode }) => {
|
||||
"resources/scss/pages/quick-search/detail.scss",
|
||||
"resources/scss/pages/quick-search/index.scss",
|
||||
"resources/scss/pages/quick-search/result.scss",
|
||||
"resources/scss/pages/pbg-task/show.scss",
|
||||
|
||||
"node_modules/quill/dist/quill.snow.css",
|
||||
"node_modules/quill/dist/quill.bubble.css",
|
||||
|
||||
Reference in New Issue
Block a user