arifal/sibedas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add new url scraping data and create tab data lists

Browse Source
This commit is contained in:
arifal
2025-08-15 17:25:20 +07:00
parent 6896fd62a3
commit 209ef07f9c
18 changed files with 1220 additions and 424 deletions
Download Patch File Download Diff File Expand all files Collapse all files

319
app/Console/Commands/InitSpatialPlanningDatas.php
View File

@@ -1,319 +0,0 @@
<?php
namespace App\Console\Commands;
use App\Models\SpatialPlanning;
use Illuminate\Console\Command;
use Maatwebsite\Excel\Facades\Excel;
use Illuminate\Support\Facades\DB;
use Exception;
class InitSpatialPlanningDatas extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'spatial:init {file? : Path to the CSV/Excel file} {--truncate : Clear existing data before import} {--safe-truncate : Clear only spatial plannings without retribution proposals} {--force-truncate : Force truncate by disabling foreign key checks}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Import spatial planning data from CSV/Excel file for retribution';
/**
* Execute the console command.
*/
public function handle()
{
$filePath = $this->argument('file') ?? 'public/templates/2025.xlsx';
$fullPath = storage_path('app/' . $filePath);
// Check if file exists
if (!file_exists($fullPath)) {
$this->error("File not found: {$fullPath}");
$this->info("Available files in templates:");
$this->listAvailableFiles();
return 1;
}
// Handle truncate options
if (($this->option('truncate') && $this->option('safe-truncate')) ||
($this->option('truncate') && $this->option('force-truncate')) ||
($this->option('safe-truncate') && $this->option('force-truncate'))) {
$this->error('Cannot use multiple truncate options together. Choose only one.');
return 1;
}
// Confirm truncate if requested
if ($this->option('truncate')) {
if ($this->confirm('This will delete all existing spatial planning data and related retribution proposals. Continue?')) {
$this->info('Truncating tables...');
try {
// First delete retribution proposals that reference spatial plannings
$deletedProposals = DB::table('retribution_proposals')
->whereNotNull('spatial_planning_id')
->count();
if ($deletedProposals > 0) {
DB::table('retribution_proposals')
->whereNotNull('spatial_planning_id')
->delete();
$this->info("Deleted {$deletedProposals} retribution proposals linked to spatial plannings.");
}
// Method 1: Try truncate with disabled foreign key checks
try {
DB::statement('SET FOREIGN_KEY_CHECKS = 0');
DB::table('spatial_plannings')->truncate();
DB::statement('SET FOREIGN_KEY_CHECKS = 1');
$this->info('Spatial plannings table truncated successfully.');
} catch (\Exception $truncateError) {
// Method 2: Fallback to delete if truncate fails
$this->warn('Truncate failed, using delete method...');
$deletedSpatial = DB::table('spatial_plannings')->delete();
$this->info("Deleted {$deletedSpatial} spatial planning records.");
// Reset auto increment
DB::statement('ALTER TABLE spatial_plannings AUTO_INCREMENT = 1');
}
} catch (\Exception $e) {
$this->error('Failed to truncate tables: ' . $e->getMessage());
return 1;
}
} else {
$this->info('Operation cancelled.');
return 0;
}
}
// Force truncate - disable foreign key checks and truncate everything
if ($this->option('force-truncate')) {
if ($this->confirm('This will FORCE truncate ALL spatial planning data by disabling foreign key checks. This is risky! Continue?')) {
$this->info('Force truncating with disabled foreign key checks...');
try {
DB::beginTransaction();
// Disable foreign key checks
DB::statement('SET FOREIGN_KEY_CHECKS = 0');
// Truncate both tables
DB::table('retribution_proposals')->truncate();
DB::table('spatial_plannings')->truncate();
// Re-enable foreign key checks
DB::statement('SET FOREIGN_KEY_CHECKS = 1');
$this->info('Force truncate completed successfully.');
DB::commit();
} catch (\Exception $e) {
DB::rollBack();
// Make sure to re-enable foreign key checks even on error
try {
DB::statement('SET FOREIGN_KEY_CHECKS = 1');
} catch (\Exception $fkError) {
$this->error('Failed to re-enable foreign key checks: ' . $fkError->getMessage());
}
$this->error('Failed to force truncate: ' . $e->getMessage());
return 1;
}
} else {
$this->info('Operation cancelled.');
return 0;
}
}
// Safe truncate - only delete spatial plannings without retribution proposals
if ($this->option('safe-truncate')) {
if ($this->confirm('This will delete only spatial planning data that have no retribution proposals. Continue?')) {
$this->info('Safe truncating spatial plannings...');
try {
DB::beginTransaction();
// Count spatial plannings with retribution proposals
$withProposals = DB::table('spatial_plannings')
->whereExists(function ($query) {
$query->select(DB::raw(1))
->from('retribution_proposals')
->whereColumn('retribution_proposals.spatial_planning_id', 'spatial_plannings.id');
})
->count();
// Delete spatial plannings without retribution proposals
$deletedCount = DB::table('spatial_plannings')
->whereNotExists(function ($query) {
$query->select(DB::raw(1))
->from('retribution_proposals')
->whereColumn('retribution_proposals.spatial_planning_id', 'spatial_plannings.id');
})
->delete();
$this->info("Deleted {$deletedCount} spatial plannings without retribution proposals.");
$this->info("Kept {$withProposals} spatial plannings that have retribution proposals.");
DB::commit();
} catch (\Exception $e) {
DB::rollBack();
$this->error('Failed to safe truncate: ' . $e->getMessage());
return 1;
}
} else {
$this->info('Operation cancelled.');
return 0;
}
}
$this->info("Starting import from: {$filePath}");
$this->info("Full path: {$fullPath}");
try {
DB::beginTransaction();
$data = Excel::toArray([], $fullPath);
if (empty($data) || empty($data[0])) {
$this->error('No data found in the file.');
return 1;
}
$rows = $data[0]; // Get first sheet
$headers = array_shift($rows); // Remove header row
$this->info("Found " . count($rows) . " data rows to import.");
$this->info("Headers: " . implode(', ', $headers));
dd($rows[0]);
$progressBar = $this->output->createProgressBar(count($rows));
$progressBar->start();
$imported = 0;
$skipped = 0;
foreach ($rows as $index => $row) {
try {
// Skip empty rows
if (empty(array_filter($row))) {
$skipped++;
$progressBar->advance();
continue;
}
// Map CSV columns to model attributes
$spatialData = [
'name' => $this->cleanString($row[1] ?? ''), // pemohon
'location' => $this->cleanString($row[2] ?? ''), // alamat
'activities' => $this->cleanString($row[3] ?? ''), // activities
'land_area' => $this->cleanNumber($row[4] ?? 0), // luas_lahan
'site_bcr' => $this->cleanNumber($row[5] ?? 0), // bcr_kawasan
'area' => $this->cleanNumber($row[6] ?? 0), // area
'no_tapak' => $this->cleanString($row[7] ?? ''), // no_tapak
'no_skkl' => $this->cleanString($row[8] ?? ''), // no_skkl
'no_ukl' => $this->cleanString($row[9] ?? ''), // no_ukl
'building_function' => $this->cleanString($row[10] ?? ''), // fungsi_bangunan
'sub_building_function' => $this->cleanString($row[11] ?? ''), // sub_fungsi_bangunan
'number_of_floors' => $this->cleanNumber($row[12] ?? 1), // jumlah_lantai
'number' => $this->cleanString($row[0] ?? ''), // no
'date' => now(), // Set current date
'kbli' => null, // Not in CSV, set as null
];
// Validate required fields
if (empty($spatialData['name']) && empty($spatialData['activities'])) {
$skipped++;
$progressBar->advance();
continue;
}
SpatialPlanning::create($spatialData);
$imported++;
} catch (Exception $e) {
$this->newLine();
$this->error("Error importing row " . ($index + 2) . ": " . $e->getMessage());
$skipped++;
}
$progressBar->advance();
}
$progressBar->finish();
$this->newLine(2);
DB::commit();
$this->info("Import completed successfully!");
$this->info("Imported: {$imported} records");
$this->info("Skipped: {$skipped} records");
return 0;
} catch (Exception $e) {
DB::rollBack();
$this->error("Import failed: " . $e->getMessage());
return 1;
}
}
/**
* Clean string data
*/
private function cleanString($value)
{
if (is_null($value)) return null;
return trim(str_replace(["\n", "\r", "\t"], ' ', $value));
}
/**
* Clean numeric data
*/
private function cleanNumber($value)
{
if (is_null($value) || $value === '') return 0;
// Remove non-numeric characters except decimal point
$cleaned = preg_replace('/[^0-9.]/', '', $value);
return is_numeric($cleaned) ? (float) $cleaned : 0;
}
/**
* List available template files
*/
private function listAvailableFiles()
{
$templatesPath = storage_path('app/public/templates');
if (is_dir($templatesPath)) {
$this->info("Files in storage/app/public/templates:");
$extensions = ['csv', 'xlsx', 'xls'];
foreach ($extensions as $ext) {
$files = glob($templatesPath . '/*.' . $ext);
foreach ($files as $file) {
$this->line(' - ' . basename($file));
}
}
}
$publicTemplatesPath = public_path('templates');
if (is_dir($publicTemplatesPath)) {
$this->info("Files in public/templates:");
$extensions = ['csv', 'xlsx', 'xls'];
foreach ($extensions as $ext) {
$files = glob($publicTemplatesPath . '/*.' . $ext);
foreach ($files as $file) {
$this->line(' - ' . basename($file));
}
}
}
}
}

45
app/Console/Commands/ScrapingData.php
View File

@@ -1,45 +0,0 @@
<?php
namespace App\Console\Commands;
use App\Jobs\ScrapingDataJob;
use App\Models\ImportDatasource;
use App\Services\ServiceGoogleSheet;
use App\Services\ServicePbgTask;
use App\Services\ServiceTabPbgTask;
use App\Services\ServiceTokenSIMBG;
use GuzzleHttp\Client; // Import Guzzle Client
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\DB;
class ScrapingData extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'app:scraping-data';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Command description';
/**
* Inject dependencies.
*/
public function __construct(
) {
parent::__construct();
}
public function handle()
{
dispatch(new ScrapingDataJob());
$this->info("Scraping job dispatched successfully");
}
}

80
app/Console/Commands/StartScrapingData.php Normal file
View File

@@ -0,0 +1,80 @@
<?php
namespace App\Console\Commands;
use App\Jobs\ScrapingDataJob;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Log;
class StartScrapingData extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'app:start-scraping-data
{--confirm : Skip confirmation prompt}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Start the optimized scraping data job (Google Sheet -> PBG Task -> Details)';
/**
* Execute the console command.
*/
public function handle()
{
$this->info('🚀 Starting Optimized Scraping Data Job');
$this->info('=====================================');
if (!$this->option('confirm')) {
$this->warn('⚠️ This will start a comprehensive data scraping process:');
$this->line(' 1. Google Sheet data scraping');
$this->line(' 2. PBG Task parent data scraping');
$this->line(' 3. Detailed task information scraping');
$this->line(' 4. BigData resume generation');
$this->newLine();
if (!$this->confirm('Do you want to continue?')) {
$this->info('Operation cancelled.');
return 0;
}
}
try {
// Dispatch the optimized job
$job = new ScrapingDataJob();
dispatch($job);
Log::info('ScrapingDataJob dispatched via command', [
'command' => $this->signature,
'user' => $this->option('confirm') ? 'auto' : 'manual'
]);
$this->info('✅ Scraping Data Job has been dispatched to the scraping queue!');
$this->newLine();
$this->info('📊 Monitor the job with:');
$this->line(' php artisan queue:monitor scraping');
$this->newLine();
$this->info('📜 View detailed logs with:');
$this->line(' tail -f /var/log/supervisor/sibedas-queue-scraping.log | grep "SCRAPING DATA JOB"');
$this->newLine();
$this->info('🔍 Check ImportDatasource status:');
$this->line(' docker-compose -f docker-compose.local.yml exec app php artisan tinker --execute="App\\Models\\ImportDatasource::latest()->first();"');
} catch (\Exception $e) {
$this->error('❌ Failed to dispatch ScrapingDataJob: ' . $e->getMessage());
Log::error('Failed to dispatch ScrapingDataJob via command', [
'error' => $e->getMessage(),
'trace' => $e->getTraceAsString()
]);
return 1;
}
return 0;
}
}