arifal/sibedas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix upload business and industry with chunk

Browse Source
This commit is contained in:
arifal
2025-02-28 00:35:09 +07:00
parent 30ca819aa1
commit 544ad1db46
4 changed files with 110 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
app/Http/Controllers/Api/BusinessOrIndustriesController.php
View File

@@ -9,6 +9,7 @@ use App\Models\BusinessOrIndustry;
use Illuminate\Http\Request;
use Maatwebsite\Excel\Facades\Excel;
use \Illuminate\Support\Facades\Validator;
use App\Http\Requests\ExcelUploadRequest;
class BusinessOrIndustriesController extends Controller
{
/**
@@ -79,29 +80,15 @@ class BusinessOrIndustriesController extends Controller
}
}
public function upload(Request $request){
if ($request->hasFile('file')) {
$file = $request->file('file');
}
// Validasi file
$validator = Validator::make($request->all(), [
'file' => 'required|mimes:xlsx,xls|max:102400', // Max 100MB
]);
if ($validator->fails()) {
public function upload(ExcelUploadRequest $request){
try {
if(!$request->hasFile('file')){
return response()->json([
'message' => 'File validation failed.',
'errors' => $validator->errors()
'error' => 'No file provided'
], 400);
}
try {
// Ambil file dari request
$file = $request->file('file');
// Menggunakan Laravel Excel untuk mengimpor file
Excel::import(new BusinessIndustriesImport, $file);
// Jika sukses, kembalikan respons sukses

114
app/Imports/BusinessIndustriesImport.php
View File

@@ -3,37 +3,105 @@
namespace App\Imports;
use App\Models\BusinessOrIndustry;
use Maatwebsite\Excel\Concerns\ToModel;
use Maatwebsite\Excel\Concerns\ToCollection;
use Illuminate\Support\Collection;
use Maatwebsite\Excel\Concerns\ToCollection;
use Maatwebsite\Excel\Concerns\WithChunkReading;
use Maatwebsite\Excel\Concerns\WithHeadingRow;
use Maatwebsite\Excel\Concerns\WithMultipleSheets;
use Illuminate\Contracts\Queue\ShouldQueue;
use Maatwebsite\Excel\Concerns\WithBatchInserts;
use Illuminate\Support\Facades\Log;
class BusinessIndustriesImport implements ToCollection
class BusinessIndustriesImport implements ToCollection, WithMultipleSheets, WithChunkReading, WithBatchInserts, ShouldQueue, WithHeadingRow
{
/**
* @param array $row
*
* @return \Illuminate\Database\Eloquent\Model|null
* @param Collection $collection
*/
public function collection(Collection $rows)
public function collection(Collection $collection)
{
foreach ($rows->skip(1) as $row){
$clean_nop = preg_replace('/[^A-Za-z0-9]/', '', $row[2]);
if (!BusinessOrIndustry::where('nop', $clean_nop)->exists()) {
BusinessOrIndustry::create([
'nama_kecamatan' => $row[0],
'nama_kelurahan' => $row[1],
'nop' => $clean_nop, // Store cleaned 'nop'
'nama_wajib_pajak' => $row[3],
'alamat_wajib_pajak' => $row[4],
'alamat_objek_pajak' => $row[5],
'luas_bumi' => $row[6],
'luas_bangunan' => $row[7],
'njop_bumi' => $row[8],
'njop_bangunan' => $row[9],
'ketetapan' => $row[10],
'tahun_pajak' => $row[11],
try{
$batchData = [];
$batchSize = 1000;
foreach ($collection as $row){
if(!isset($row['nop']) || empty($row['nop'])){
continue;
}
$clean_nop = preg_replace('/[^A-Za-z0-9]/', '', $row['nop']);
$batchData[] = [
'nama_kecamatan' => $row['nama_kecamatan'],
'nama_kelurahan' => $row['nama_kelurahan'],
'nop' => $clean_nop,
'nama_wajib_pajak' => $row['nama_wajib_pajak'],
'alamat_wajib_pajak' => $row['alamat_wajib_pajak'],
'alamat_objek_pajak' => $row['alamat_objek_pajak'],
'luas_bumi' => $row['luas_bumi'],
'luas_bangunan' => $row['luas_bangunan'],
'njop_bumi' => $row['njop_bumi'],
'njop_bangunan' => $row['njop_bangunan'],
'ketetapan' => $row['ketetapan'],
'tahun_pajak' => $row['tahun_pajak'],
];
if(count($batchData) >= $batchSize){
BusinessOrIndustry::upsert($batchData, ['nop'], [
'nama_kecamatan',
'nama_kelurahan',
'nama_wajib_pajak',
'alamat_wajib_pajak',
'alamat_objek_pajak',
'luas_bumi',
'luas_bangunan',
'njop_bumi',
'njop_bangunan',
'ketetapan',
'tahun_pajak',
]);
$batchData = [];
}
}
if(!empty($batchData)){
BusinessOrIndustry::upsert($batchData, ['nop'], [
'nama_kecamatan',
'nama_kelurahan',
'nama_wajib_pajak',
'alamat_wajib_pajak',
'alamat_objek_pajak',
'luas_bumi',
'luas_bangunan',
'njop_bumi',
'njop_bangunan',
'ketetapan',
'tahun_pajak',
]);
}
}catch(\Exception $exception){
Log::error('Error while importing Business Industries data:', ['error' => $exception->getMessage()]);
return;
}
}
public function sheets(): array {
return [
0 => $this
];
}
public function headingRow(): int
{
return 1;
}
public function chunkSize(): int
{
return 1000;
}
public function batchSize(): int
{
return 1000;
}
}

23
app/Imports/CustomersImport.php
View File

@@ -21,36 +21,33 @@ class CustomersImport implements ToCollection, WithMultipleSheets, WithChunkRead
public function collection(Collection $collection)
{
$batchData = [];
$batchSize = 1000; // Process in smaller chunks
$batchSize = 1000;
foreach ($collection as $row) {
if (!isset($row['nomor_pelanggan']) || empty($row['nomor_pelanggan'])) {
continue; // Skip rows without 'nomor_pelanggan'
continue;
}
// Default values
$latitude = '0';
$longitude = '0';
// Convert and normalize latitude
if (isset($row['latkor']) && !empty(trim($row['latkor']))) {
$latitude = str_replace(',', '.', trim($row['latkor'])); // Replace comma with dot
$latitude = str_replace(',', '.', trim($row['latkor']));
if (is_numeric($latitude)) {
$latitude = bcadd($latitude, '0', 18); // Convert to decimal with 18 precision
$latitude = bcadd($latitude, '0', 18);
} else {
$latitude = '0'; // Default fallback
$latitude = '0';
}
} else {
$latitude = '0';
}
// Convert and normalize longitude
if (isset($row['lonkor']) && !empty(trim($row['lonkor']))) {
$longitude = str_replace(',', '.', trim($row['lonkor'])); // Replace comma with dot
$longitude = str_replace(',', '.', trim($row['lonkor']));
if (is_numeric($longitude)) {
$longitude = bcadd($longitude, '0', 18); // Convert to decimal with 18 precision
$longitude = bcadd($longitude, '0', 18);
} else {
$longitude = '0'; // Default fallback
$longitude = '0';
}
} else {
$longitude = '0';
@@ -65,14 +62,12 @@ class CustomersImport implements ToCollection, WithMultipleSheets, WithChunkRead
'longitude' => $longitude,
];
// Batch insert every 1000 rows
if (count($batchData) >= $batchSize) {
Customer::upsert($batchData, ['nomor_pelanggan'], ['kota_pelayanan', 'nama', 'alamat', 'latitude', 'longitude']);
$batchData = []; // Clear the batch
$batchData = [];
}
}
// Insert remaining data
if (!empty($batchData)) {
Customer::upsert($batchData, ['nomor_pelanggan'], ['kota_pelayanan', 'nama', 'alamat', 'latitude', 'longitude']);
}

2
composer.json
View File

@@ -56,7 +56,7 @@
],
"dev": [
"Composer\\Config::disableProcessTimeout",
"npx concurrently -c \"#93c5fd,#c4b5fd,#fb7185,#fdba74\" \"php artisan serve\" \"php artisan pail --timeout=0\" \"npm run dev\" --names=server,queue,logs,vite"
"npx concurrently -c \"#93c5fd,#c4b5fd,#fb7185,#fdba74\" \"php artisan serve\" \"php artisan queue:listen --tries=1\" \"php artisan pail --timeout=0\" \"npm run dev\" --names=server,queue,logs,vite"
]
},
"extra": {