From 544ad1db462e6cdb2008025901f32407be61210b Mon Sep 17 00:00:00 2001 From: arifal Date: Fri, 28 Feb 2025 00:35:09 +0700 Subject: [PATCH] fix upload business and industry with chunk --- .../Api/BusinessOrIndustriesController.php | 29 ++--- app/Imports/BusinessIndustriesImport.php | 116 ++++++++++++++---- app/Imports/CustomersImport.php | 23 ++-- composer.json | 2 +- 4 files changed, 110 insertions(+), 60 deletions(-) diff --git a/app/Http/Controllers/Api/BusinessOrIndustriesController.php b/app/Http/Controllers/Api/BusinessOrIndustriesController.php index dc6eb57..d426b0b 100644 --- a/app/Http/Controllers/Api/BusinessOrIndustriesController.php +++ b/app/Http/Controllers/Api/BusinessOrIndustriesController.php @@ -9,6 +9,7 @@ use App\Models\BusinessOrIndustry; use Illuminate\Http\Request; use Maatwebsite\Excel\Facades\Excel; use \Illuminate\Support\Facades\Validator; +use App\Http\Requests\ExcelUploadRequest; class BusinessOrIndustriesController extends Controller { /** @@ -79,29 +80,15 @@ class BusinessOrIndustriesController extends Controller } } - public function upload(Request $request){ - - if ($request->hasFile('file')) { - $file = $request->file('file'); - } - - // Validasi file - $validator = Validator::make($request->all(), [ - 'file' => 'required|mimes:xlsx,xls|max:102400', // Max 100MB - ]); - - if ($validator->fails()) { - return response()->json([ - 'message' => 'File validation failed.', - 'errors' => $validator->errors() - ], 400); - } - + public function upload(ExcelUploadRequest $request){ try { - // Ambil file dari request - $file = $request->file('file'); + if(!$request->hasFile('file')){ + return response()->json([ + 'error' => 'No file provided' + ], 400); + } - // Menggunakan Laravel Excel untuk mengimpor file + $file = $request->file('file'); Excel::import(new BusinessIndustriesImport, $file); // Jika sukses, kembalikan respons sukses diff --git a/app/Imports/BusinessIndustriesImport.php b/app/Imports/BusinessIndustriesImport.php index 3cdc88d..0ade79b 100644 --- a/app/Imports/BusinessIndustriesImport.php +++ b/app/Imports/BusinessIndustriesImport.php @@ -3,37 +3,105 @@ namespace App\Imports; use App\Models\BusinessOrIndustry; -use Maatwebsite\Excel\Concerns\ToModel; -use Maatwebsite\Excel\Concerns\ToCollection; use Illuminate\Support\Collection; +use Maatwebsite\Excel\Concerns\ToCollection; +use Maatwebsite\Excel\Concerns\WithChunkReading; +use Maatwebsite\Excel\Concerns\WithHeadingRow; +use Maatwebsite\Excel\Concerns\WithMultipleSheets; +use Illuminate\Contracts\Queue\ShouldQueue; +use Maatwebsite\Excel\Concerns\WithBatchInserts; +use Illuminate\Support\Facades\Log; -class BusinessIndustriesImport implements ToCollection +class BusinessIndustriesImport implements ToCollection, WithMultipleSheets, WithChunkReading, WithBatchInserts, ShouldQueue, WithHeadingRow { /** - * @param array $row - * - * @return \Illuminate\Database\Eloquent\Model|null - */ - public function collection(Collection $rows) + * @param Collection $collection + */ + public function collection(Collection $collection) { - foreach ($rows->skip(1) as $row){ - $clean_nop = preg_replace('/[^A-Za-z0-9]/', '', $row[2]); - if (!BusinessOrIndustry::where('nop', $clean_nop)->exists()) { - BusinessOrIndustry::create([ - 'nama_kecamatan' => $row[0], - 'nama_kelurahan' => $row[1], - 'nop' => $clean_nop, // Store cleaned 'nop' - 'nama_wajib_pajak' => $row[3], - 'alamat_wajib_pajak' => $row[4], - 'alamat_objek_pajak' => $row[5], - 'luas_bumi' => $row[6], - 'luas_bangunan' => $row[7], - 'njop_bumi' => $row[8], - 'njop_bangunan' => $row[9], - 'ketetapan' => $row[10], - 'tahun_pajak' => $row[11], + try{ + $batchData = []; + $batchSize = 1000; + + foreach ($collection as $row){ + if(!isset($row['nop']) || empty($row['nop'])){ + continue; + } + + + $clean_nop = preg_replace('/[^A-Za-z0-9]/', '', $row['nop']); + + $batchData[] = [ + 'nama_kecamatan' => $row['nama_kecamatan'], + 'nama_kelurahan' => $row['nama_kelurahan'], + 'nop' => $clean_nop, + 'nama_wajib_pajak' => $row['nama_wajib_pajak'], + 'alamat_wajib_pajak' => $row['alamat_wajib_pajak'], + 'alamat_objek_pajak' => $row['alamat_objek_pajak'], + 'luas_bumi' => $row['luas_bumi'], + 'luas_bangunan' => $row['luas_bangunan'], + 'njop_bumi' => $row['njop_bumi'], + 'njop_bangunan' => $row['njop_bangunan'], + 'ketetapan' => $row['ketetapan'], + 'tahun_pajak' => $row['tahun_pajak'], + ]; + + if(count($batchData) >= $batchSize){ + BusinessOrIndustry::upsert($batchData, ['nop'], [ + 'nama_kecamatan', + 'nama_kelurahan', + 'nama_wajib_pajak', + 'alamat_wajib_pajak', + 'alamat_objek_pajak', + 'luas_bumi', + 'luas_bangunan', + 'njop_bumi', + 'njop_bangunan', + 'ketetapan', + 'tahun_pajak', + ]); + $batchData = []; + } + } + if(!empty($batchData)){ + BusinessOrIndustry::upsert($batchData, ['nop'], [ + 'nama_kecamatan', + 'nama_kelurahan', + 'nama_wajib_pajak', + 'alamat_wajib_pajak', + 'alamat_objek_pajak', + 'luas_bumi', + 'luas_bangunan', + 'njop_bumi', + 'njop_bangunan', + 'ketetapan', + 'tahun_pajak', ]); } + }catch(\Exception $exception){ + Log::error('Error while importing Business Industries data:', ['error' => $exception->getMessage()]); + return; } } + + public function sheets(): array { + return [ + 0 => $this + ]; + } + + public function headingRow(): int + { + return 1; + } + + public function chunkSize(): int + { + return 1000; + } + + public function batchSize(): int + { + return 1000; + } } diff --git a/app/Imports/CustomersImport.php b/app/Imports/CustomersImport.php index dff0824..7f25b65 100644 --- a/app/Imports/CustomersImport.php +++ b/app/Imports/CustomersImport.php @@ -21,36 +21,33 @@ class CustomersImport implements ToCollection, WithMultipleSheets, WithChunkRead public function collection(Collection $collection) { $batchData = []; - $batchSize = 1000; // Process in smaller chunks + $batchSize = 1000; foreach ($collection as $row) { if (!isset($row['nomor_pelanggan']) || empty($row['nomor_pelanggan'])) { - continue; // Skip rows without 'nomor_pelanggan' + continue; } - // Default values $latitude = '0'; $longitude = '0'; - // Convert and normalize latitude if (isset($row['latkor']) && !empty(trim($row['latkor']))) { - $latitude = str_replace(',', '.', trim($row['latkor'])); // Replace comma with dot + $latitude = str_replace(',', '.', trim($row['latkor'])); if (is_numeric($latitude)) { - $latitude = bcadd($latitude, '0', 18); // Convert to decimal with 18 precision + $latitude = bcadd($latitude, '0', 18); } else { - $latitude = '0'; // Default fallback + $latitude = '0'; } } else { $latitude = '0'; } - // Convert and normalize longitude if (isset($row['lonkor']) && !empty(trim($row['lonkor']))) { - $longitude = str_replace(',', '.', trim($row['lonkor'])); // Replace comma with dot + $longitude = str_replace(',', '.', trim($row['lonkor'])); if (is_numeric($longitude)) { - $longitude = bcadd($longitude, '0', 18); // Convert to decimal with 18 precision + $longitude = bcadd($longitude, '0', 18); } else { - $longitude = '0'; // Default fallback + $longitude = '0'; } } else { $longitude = '0'; @@ -65,14 +62,12 @@ class CustomersImport implements ToCollection, WithMultipleSheets, WithChunkRead 'longitude' => $longitude, ]; - // Batch insert every 1000 rows if (count($batchData) >= $batchSize) { Customer::upsert($batchData, ['nomor_pelanggan'], ['kota_pelayanan', 'nama', 'alamat', 'latitude', 'longitude']); - $batchData = []; // Clear the batch + $batchData = []; } } - // Insert remaining data if (!empty($batchData)) { Customer::upsert($batchData, ['nomor_pelanggan'], ['kota_pelayanan', 'nama', 'alamat', 'latitude', 'longitude']); } diff --git a/composer.json b/composer.json index 6af873f..755703d 100755 --- a/composer.json +++ b/composer.json @@ -56,7 +56,7 @@ ], "dev": [ "Composer\\Config::disableProcessTimeout", - "npx concurrently -c \"#93c5fd,#c4b5fd,#fb7185,#fdba74\" \"php artisan serve\" \"php artisan pail --timeout=0\" \"npm run dev\" --names=server,queue,logs,vite" + "npx concurrently -c \"#93c5fd,#c4b5fd,#fb7185,#fdba74\" \"php artisan serve\" \"php artisan queue:listen --tries=1\" \"php artisan pail --timeout=0\" \"npm run dev\" --names=server,queue,logs,vite" ] }, "extra": {