File tree Expand file tree Collapse file tree 1 file changed +13
-0
lines changed
python/sglang/srt/mem_cache Expand file tree Collapse file tree 1 file changed +13
-0
lines changed Original file line number Diff line number Diff line change @@ -442,6 +442,19 @@ def __init__(
442
442
443
443
self .layer_transfer_counter = None
444
444
445
+ # for disagg
446
+ def get_contiguous_buf_infos (self ):
447
+ kv_data_ptrs = [
448
+ self .get_key_buffer (i ).data_ptr () for i in range (self .layer_num )
449
+ ] + [self .get_value_buffer (i ).data_ptr () for i in range (self .layer_num )]
450
+ kv_data_lens = [
451
+ self .get_key_buffer (i ).nbytes for i in range (self .layer_num )
452
+ ] + [self .get_value_buffer (i ).nbytes for i in range (self .layer_num )]
453
+ kv_item_lens = [
454
+ self .get_key_buffer (i )[0 ].nbytes for i in range (self .layer_num )
455
+ ] + [self .get_value_buffer (i )[0 ].nbytes for i in range (self .layer_num )]
456
+ return kv_data_ptrs , kv_data_lens , kv_item_lens
457
+
445
458
def get_key_buffer (self , layer_id : int ):
446
459
if self .layer_transfer_counter is not None :
447
460
self .layer_transfer_counter .wait_until (layer_id )
You can’t perform that action at this time.
0 commit comments