arifal/sibedas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix upload business and industry with chunk

Browse Source
This commit is contained in:
arifal
2025-02-28 00:35:09 +07:00
parent 30ca819aa1
commit 544ad1db46
4 changed files with 110 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
app/Http/Controllers/Api/BusinessOrIndustriesController.php
View File

@@ -9,6 +9,7 @@ use App\Models\BusinessOrIndustry;
use Illuminate\Http\Request; use Illuminate\Http\Request;
use Maatwebsite\Excel\Facades\Excel; use Maatwebsite\Excel\Facades\Excel;
use \Illuminate\Support\Facades\Validator; use \Illuminate\Support\Facades\Validator;
use App\Http\Requests\ExcelUploadRequest;
class BusinessOrIndustriesController extends Controller class BusinessOrIndustriesController extends Controller
{ {
/** /**
@@ -79,29 +80,15 @@ class BusinessOrIndustriesController extends Controller
} }
} }
public function upload(Request $request){ public function upload(ExcelUploadRequest $request){
try {
if ($request->hasFile('file')) { if(!$request->hasFile('file')){
$file = $request->file('file');
}
// Validasi file
$validator = Validator::make($request->all(), [
'file' => 'required|mimes:xlsx,xls|max:102400', // Max 100MB
]);
if ($validator->fails()) {
return response()->json([ return response()->json([
'message' => 'File validation failed.', 'error' => 'No file provided'
'errors' => $validator->errors()
], 400); ], 400);
} }
try {
// Ambil file dari request
$file = $request->file('file'); $file = $request->file('file');
// Menggunakan Laravel Excel untuk mengimpor file
Excel::import(new BusinessIndustriesImport, $file); Excel::import(new BusinessIndustriesImport, $file);
// Jika sukses, kembalikan respons sukses // Jika sukses, kembalikan respons sukses

114
app/Imports/BusinessIndustriesImport.php
View File

@@ -3,37 +3,105 @@
namespace App\Imports; namespace App\Imports;
use App\Models\BusinessOrIndustry; use App\Models\BusinessOrIndustry;
use Maatwebsite\Excel\Concerns\ToModel;
use Maatwebsite\Excel\Concerns\ToCollection;
use Illuminate\Support\Collection; use Illuminate\Support\Collection;
use Maatwebsite\Excel\Concerns\ToCollection;
use Maatwebsite\Excel\Concerns\WithChunkReading;
use Maatwebsite\Excel\Concerns\WithHeadingRow;
use Maatwebsite\Excel\Concerns\WithMultipleSheets;
use Illuminate\Contracts\Queue\ShouldQueue;
use Maatwebsite\Excel\Concerns\WithBatchInserts;
use Illuminate\Support\Facades\Log;
class BusinessIndustriesImport implements ToCollection class BusinessIndustriesImport implements ToCollection, WithMultipleSheets, WithChunkReading, WithBatchInserts, ShouldQueue, WithHeadingRow
{ {
/** /**
* @param array $row * @param Collection $collection
*
* @return \Illuminate\Database\Eloquent\Model|null
*/ */
public function collection(Collection $rows) public function collection(Collection $collection)
{ {
foreach ($rows->skip(1) as $row){ try{
$clean_nop = preg_replace('/[^A-Za-z0-9]/', '', $row[2]); $batchData = [];
if (!BusinessOrIndustry::where('nop', $clean_nop)->exists()) { $batchSize = 1000;
BusinessOrIndustry::create([
'nama_kecamatan' => $row[0], foreach ($collection as $row){
'nama_kelurahan' => $row[1], if(!isset($row['nop']) || empty($row['nop'])){
'nop' => $clean_nop, // Store cleaned 'nop' continue;
'nama_wajib_pajak' => $row[3], }
'alamat_wajib_pajak' => $row[4],
'alamat_objek_pajak' => $row[5],
'luas_bumi' => $row[6], $clean_nop = preg_replace('/[^A-Za-z0-9]/', '', $row['nop']);
'luas_bangunan' => $row[7],
'njop_bumi' => $row[8], $batchData[] = [
'njop_bangunan' => $row[9], 'nama_kecamatan' => $row['nama_kecamatan'],
'ketetapan' => $row[10], 'nama_kelurahan' => $row['nama_kelurahan'],
'tahun_pajak' => $row[11], 'nop' => $clean_nop,
'nama_wajib_pajak' => $row['nama_wajib_pajak'],
'alamat_wajib_pajak' => $row['alamat_wajib_pajak'],
'alamat_objek_pajak' => $row['alamat_objek_pajak'],
'luas_bumi' => $row['luas_bumi'],
'luas_bangunan' => $row['luas_bangunan'],
'njop_bumi' => $row['njop_bumi'],
'njop_bangunan' => $row['njop_bangunan'],
'ketetapan' => $row['ketetapan'],
'tahun_pajak' => $row['tahun_pajak'],
];
if(count($batchData) >= $batchSize){
BusinessOrIndustry::upsert($batchData, ['nop'], [
'nama_kecamatan',
'nama_kelurahan',
'nama_wajib_pajak',
'alamat_wajib_pajak',
'alamat_objek_pajak',
'luas_bumi',
'luas_bangunan',
'njop_bumi',
'njop_bangunan',
'ketetapan',
'tahun_pajak',
]);
$batchData = [];
}
}
if(!empty($batchData)){
BusinessOrIndustry::upsert($batchData, ['nop'], [
'nama_kecamatan',
'nama_kelurahan',
'nama_wajib_pajak',
'alamat_wajib_pajak',
'alamat_objek_pajak',
'luas_bumi',
'luas_bangunan',
'njop_bumi',
'njop_bangunan',
'ketetapan',
'tahun_pajak',
]); ]);
} }
}catch(\Exception $exception){
Log::error('Error while importing Business Industries data:', ['error' => $exception->getMessage()]);
return;
} }
} }
public function sheets(): array {
return [
0 => $this
];
}
public function headingRow(): int
{
return 1;
}
public function chunkSize(): int
{
return 1000;
}
public function batchSize(): int
{
return 1000;
}
} }

23
app/Imports/CustomersImport.php
View File

@@ -21,36 +21,33 @@ class CustomersImport implements ToCollection, WithMultipleSheets, WithChunkRead
public function collection(Collection $collection) public function collection(Collection $collection)
{ {
$batchData = []; $batchData = [];
$batchSize = 1000; // Process in smaller chunks $batchSize = 1000;
foreach ($collection as $row) { foreach ($collection as $row) {
if (!isset($row['nomor_pelanggan']) || empty($row['nomor_pelanggan'])) { if (!isset($row['nomor_pelanggan']) || empty($row['nomor_pelanggan'])) {
continue; // Skip rows without 'nomor_pelanggan' continue;
} }
// Default values
$latitude = '0'; $latitude = '0';
$longitude = '0'; $longitude = '0';
// Convert and normalize latitude
if (isset($row['latkor']) && !empty(trim($row['latkor']))) { if (isset($row['latkor']) && !empty(trim($row['latkor']))) {
$latitude = str_replace(',', '.', trim($row['latkor'])); // Replace comma with dot $latitude = str_replace(',', '.', trim($row['latkor']));
if (is_numeric($latitude)) { if (is_numeric($latitude)) {
$latitude = bcadd($latitude, '0', 18); // Convert to decimal with 18 precision $latitude = bcadd($latitude, '0', 18);
} else { } else {
$latitude = '0'; // Default fallback $latitude = '0';
} }
} else { } else {
$latitude = '0'; $latitude = '0';
} }
// Convert and normalize longitude
if (isset($row['lonkor']) && !empty(trim($row['lonkor']))) { if (isset($row['lonkor']) && !empty(trim($row['lonkor']))) {
$longitude = str_replace(',', '.', trim($row['lonkor'])); // Replace comma with dot $longitude = str_replace(',', '.', trim($row['lonkor']));
if (is_numeric($longitude)) { if (is_numeric($longitude)) {
$longitude = bcadd($longitude, '0', 18); // Convert to decimal with 18 precision $longitude = bcadd($longitude, '0', 18);
} else { } else {
$longitude = '0'; // Default fallback $longitude = '0';
} }
} else { } else {
$longitude = '0'; $longitude = '0';
@@ -65,14 +62,12 @@ class CustomersImport implements ToCollection, WithMultipleSheets, WithChunkRead
'longitude' => $longitude, 'longitude' => $longitude,
]; ];
// Batch insert every 1000 rows
if (count($batchData) >= $batchSize) { if (count($batchData) >= $batchSize) {
Customer::upsert($batchData, ['nomor_pelanggan'], ['kota_pelayanan', 'nama', 'alamat', 'latitude', 'longitude']); Customer::upsert($batchData, ['nomor_pelanggan'], ['kota_pelayanan', 'nama', 'alamat', 'latitude', 'longitude']);
$batchData = []; // Clear the batch $batchData = [];
} }
} }
// Insert remaining data
if (!empty($batchData)) { if (!empty($batchData)) {
Customer::upsert($batchData, ['nomor_pelanggan'], ['kota_pelayanan', 'nama', 'alamat', 'latitude', 'longitude']); Customer::upsert($batchData, ['nomor_pelanggan'], ['kota_pelayanan', 'nama', 'alamat', 'latitude', 'longitude']);
} }

2
composer.json
View File

@@ -56,7 +56,7 @@
], ],
"dev": [ "dev": [
"Composer\\Config::disableProcessTimeout", "Composer\\Config::disableProcessTimeout",
"npx concurrently -c \"#93c5fd,#c4b5fd,#fb7185,#fdba74\" \"php artisan serve\" \"php artisan pail --timeout=0\" \"npm run dev\" --names=server,queue,logs,vite" "npx concurrently -c \"#93c5fd,#c4b5fd,#fb7185,#fdba74\" \"php artisan serve\" \"php artisan queue:listen --tries=1\" \"php artisan pail --timeout=0\" \"npm run dev\" --names=server,queue,logs,vite"
] ]
}, },
"extra": { "extra": {