Skip to content

Commit a85f762

Browse files
stmatengssthyecust
authored andcommitted
[PD] Add get_contiguous_buf_infos interface for MLATokenToKVPool (sgl-project#5204)
1 parent 796073f commit a85f762

File tree

1 file changed

+13
-0
lines changed

1 file changed

+13
-0
lines changed

python/sglang/srt/mem_cache/memory_pool.py

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -442,6 +442,19 @@ def __init__(
442442

443443
self.layer_transfer_counter = None
444444

445+
# for disagg
446+
def get_contiguous_buf_infos(self):
447+
kv_data_ptrs = [
448+
self.get_key_buffer(i).data_ptr() for i in range(self.layer_num)
449+
] + [self.get_value_buffer(i).data_ptr() for i in range(self.layer_num)]
450+
kv_data_lens = [
451+
self.get_key_buffer(i).nbytes for i in range(self.layer_num)
452+
] + [self.get_value_buffer(i).nbytes for i in range(self.layer_num)]
453+
kv_item_lens = [
454+
self.get_key_buffer(i)[0].nbytes for i in range(self.layer_num)
455+
] + [self.get_value_buffer(i)[0].nbytes for i in range(self.layer_num)]
456+
return kv_data_ptrs, kv_data_lens, kv_item_lens
457+
445458
def get_key_buffer(self, layer_id: int):
446459
if self.layer_transfer_counter is not None:
447460
self.layer_transfer_counter.wait_until(layer_id)

0 commit comments

Comments
 (0)