WIP - REBASE AWAY

Author: macneale4

go/store/nbs/table_reader.go

@@ -769,74 +769,82 @@ func (tr tableReader) iterateAllChunks(ctx context.Context, cb func(chunk chunks
 		return nil
 	}
 
-	// Build cumulative offsets array for efficient chunk location
-	cumulativeOffsets := make([]uint64, count+1)
-	cumulativeOffsets[0] = 0
+	// Build offset records similar to the extract method
+	// The index is sorted by prefix, but we need to process chunkRecs in storage order (by offset)
+	type chunkRecord struct {
+		offset uint64
+		length uint32
+		hash   hash.Hash
+	}
+
+	chunkRecs := make([]chunkRecord, 0, count)
+
+	// First pass: collect all chunk info from the sorted index
 	for i := uint32(0); i < count; i++ {
-		ie, err := tr.idx.indexEntry(i, nil)
+		var h hash.Hash
+		ie, err := tr.idx.indexEntry(i, &h)
 		if err != nil {
 			return err
 		}
-		cumulativeOffsets[i+1] = cumulativeOffsets[i] + uint64(ie.Length())
+
+		chunkRecs = append(chunkRecs, chunkRecord{
+			offset: ie.Offset(),
+			length: ie.Length(),
+			hash:   h,
+		})
 	}
+	sort.Slice(chunkRecs, func(i, j int) bool {
+		return chunkRecs[i].offset < chunkRecs[j].offset
+	})
 
-	// Calculate total data size to determine where chunk records end
-	totalDataSize := cumulativeOffsets[count]
+	lastChunk := chunkRecs[len(chunkRecs)-1]
+	totalDataSize := lastChunk.offset + uint64(lastChunk.length)
 
-	// Read data in 1MB chunks
+	// Read data in 1MB chunkRecs
 	const bufferSize = 1024 * 1024 // 1MB
 	currentOffset := uint64(0)
-	chunkCounter := uint32(0)
+	chunkIndex := 0
 
 	// Reuse buffer across reads
 	dataBlock := make([]byte, bufferSize)
 
-	for currentOffset < totalDataSize {
-		// Calculate how much data to read (up to 1MB or remaining data)
+	for currentOffset < totalDataSize && chunkIndex < len(chunkRecs) {
+		// Calculate how much data to read
 		remainingData := totalDataSize - currentOffset
 		readSize := bufferSize
 		if remainingData < bufferSize {
 			readSize = int(remainingData)
-			dataBlock = dataBlock[:readSize] // Resize to remaining data
+			dataBlock = dataBlock[:readSize]
 		}
 
 		_, err := tr.r.ReadAtWithStats(ctx, dataBlock, int64(currentOffset), stats)
 		if err != nil {
 			return err
 		}
 
-		// Process all chunks that are fully contained within this data block
 		blockStart := currentOffset
 		blockEnd := currentOffset + uint64(readSize)
 
-		for chunkCounter < count {
+		// Process all chunkRecs that are fully contained within this block
+		for chunkIndex < len(chunkRecs) {
 			if ctx.Err() != nil {
 				return ctx.Err()
 			}
 
-			chunkStart := cumulativeOffsets[chunkCounter]
-			chunkEnd := cumulativeOffsets[chunkCounter+1]
+			chunk := chunkRecs[chunkIndex]
+			chunkEnd := chunk.offset + uint64(chunk.length)
 
-			// Check if this chunk extends beyond the current block
+			// Check if chunk extends beyond current block
 			if chunkEnd > blockEnd {
-				// This chunk extends beyond current block, read next block
-				currentOffset = chunkStart
+				// This chunk extends beyond current block, read next block starting here
+				currentOffset = chunk.offset
 				break
 			}
 
-			// Extract chunk data from buffer
-			bufferOffset := chunkStart - blockStart
-			chunkData := dataBlock[bufferOffset : bufferOffset+(chunkEnd-chunkStart)]
-
-			// Get the hash for this chunk
-			var h hash.Hash
-			_, err := tr.idx.indexEntry(chunkCounter, &h)
-			if err != nil {
-				return err
-			}
-
-			// Create compressed chunk and decompress
-			cchk, err := NewCompressedChunk(h, chunkData)
+			bufferOffset := chunk.offset - blockStart
+			chunkData := make([]byte, chunk.length)
+			copy(chunkData, dataBlock[bufferOffset:bufferOffset+uint64(chunk.length)])
+			cchk, err := NewCompressedChunk(chunk.hash, chunkData)
 			if err != nil {
 				return err
 			}
@@ -848,15 +856,12 @@ func (tr tableReader) iterateAllChunks(ctx context.Context, cb func(chunk chunks
 			// Process the chunk
 			cb(chk)
 
-			chunkCounter++
+			chunkIndex++
 		}
 
-		// If we processed all chunks in this block, move to next block
-		if chunkCounter < count && cumulativeOffsets[chunkCounter] >= blockEnd {
+		// If we didn't break out due to a chunk boundary, move to next block
+		if chunkIndex < len(chunkRecs) && chunkRecs[chunkIndex].offset < blockEnd {
 			currentOffset = blockEnd
-		} else if chunkCounter >= count {
-			// All chunks processed
-			break
 		}
 	}