arifal/sibedas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix duplicate insert when chunk

Browse Source
This commit is contained in:
arifal hidayat
2025-09-14 20:25:36 +07:00
parent 61e6eb9803
commit 91085e8796
1 changed files with 9 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
app/Services/ServiceGoogleSheet.php
View File

@@ -688,21 +688,27 @@ class ServiceGoogleSheet
// Build and insert in small batches to avoid high memory usage // Build and insert in small batches to avoid high memory usage
$batch = []; $batch = [];
$batchSize = 500;
$inserted = 0; $inserted = 0;
// Stream rows in chunks from API to avoid loading full sheet // Stream rows in chunks from API to avoid loading full sheet
$rowStart = 2; // data starts from row 2 $rowStart = 2; // data starts from row 2
$chunkRowSize = 800; // number of rows per chunk $chunkRowSize = 1000; // number of rows per chunk
$inserted = 0; $inserted = 0;
while (true) { while (true) {
$rowEnd = $rowStart + $chunkRowSize - 1; $rowEnd = $rowStart + $chunkRowSize - 1;
$range = sprintf('%s!%s%d:%s%d', $sheetName, $startLetter, $rowStart, 'BF', $rowEnd); $range = sprintf('%s!%s%d:%s%d', $sheetName, $startLetter, $rowStart, 'BF', $rowEnd);
$resp = $this->service->spreadsheets_values->get($this->spreadsheetID, $range); $resp = $this->service->spreadsheets_values->get($this->spreadsheetID, $range);
$values = $resp->getValues() ?? []; $values = $resp->getValues() ?? [];
if (empty($values)) { if (empty($values)) {
break; // no more rows break; // no more rows
} }
Log::info('Chunk fetched', [
'rowStart' => $rowStart,
'rowEnd' => $rowEnd,
'count' => count($values)
]);
// Preload registration map for this chunk // Preload registration map for this chunk
$chunkRegs = []; $chunkRegs = [];
foreach ($values as $row) { foreach ($values as $row) {
@@ -838,11 +844,6 @@ class ServiceGoogleSheet
if (function_exists('gc_collect_cycles')) { gc_collect_cycles(); } if (function_exists('gc_collect_cycles')) { gc_collect_cycles(); }
} }
if (!empty($batch)) {
\App\Models\PbgTaskPayment::insert($batch);
$inserted += count($batch);
}
Log::info('PBG Task Payments reloaded from sheet', ['inserted' => $inserted]); Log::info('PBG Task Payments reloaded from sheet', ['inserted' => $inserted]);
return ['success' => true, 'inserted' => $inserted]; return ['success' => true, 'inserted' => $inserted];