vllm.entrypoints.rl.offline

Classes:

• RLOfflineMixin –

RLOfflineMixin

Bases: OfflineInferenceMixin

Methods:

• finish_weight_update –

  Finish the weight update and set its version if provided.

• get_weight_version –

  Return the latest committed weight version.

• init_weight_transfer_engine –

  Initialize weight transfer for RL training.

• start_draft_weight_update –

  Start a new weight update targeting the speculative draft model.

• start_weight_update –

  Start a new weight update.

• update_weight_version –

  Set the weight version without updating weights.

• update_weights –

  Update the weights of the model.

Source code in vllm/entrypoints/rl/offline.py

class RLOfflineMixin(OfflineInferenceMixin):
    def init_weight_transfer_engine(
        self, request: WeightTransferInitRequest | dict
    ) -> None:
        """Initialize weight transfer for RL training.

        Args:
            request: Weight transfer initialization request with backend-specific info

        """
        init_info_dict = (
            request["init_info"] if isinstance(request, dict) else request.init_info
        )

        self.llm_engine.collective_rpc(
            "init_weight_transfer_engine", kwargs={"init_info": init_info_dict}
        )

    def start_weight_update(self) -> None:
        """Start a new weight update."""
        self.llm_engine.collective_rpc("start_weight_update")

    def start_draft_weight_update(self) -> None:
        """Start a new weight update targeting the speculative draft model."""
        self.llm_engine.collective_rpc("start_draft_weight_update")

    def update_weights(self, request: WeightTransferUpdateRequest | dict) -> None:
        """Update the weights of the model.

        Args:
            request: Weight update request with backend-specific update info

        """
        update_info_dict = (
            request["update_info"] if isinstance(request, dict) else request.update_info
        )

        self.llm_engine.collective_rpc(
            "update_weights", kwargs={"update_info": update_info_dict}
        )

    def finish_weight_update(self, weight_version: str | None = None) -> None:
        """Finish the weight update and set its version if provided."""
        self.llm_engine.collective_rpc("finish_weight_update")
        if weight_version is not None:
            self.llm_engine.set_weight_version(weight_version)

    def update_weight_version(self, new_version: str) -> None:
        """Set the weight version without updating weights."""
        self.llm_engine.set_weight_version(new_version)

    def get_weight_version(self) -> str:
        """Return the latest committed weight version."""
        return self.llm_engine.get_weight_version()

finish_weight_update(weight_version=None)

Finish the weight update and set its version if provided.

Source code in vllm/entrypoints/rl/offline.py

def finish_weight_update(self, weight_version: str | None = None) -> None:
    """Finish the weight update and set its version if provided."""
    self.llm_engine.collective_rpc("finish_weight_update")
    if weight_version is not None:
        self.llm_engine.set_weight_version(weight_version)

get_weight_version()

Return the latest committed weight version.

Source code in vllm/entrypoints/rl/offline.py

def get_weight_version(self) -> str:
    """Return the latest committed weight version."""
    return self.llm_engine.get_weight_version()

init_weight_transfer_engine(request)

Initialize weight transfer for RL training.

Parameters:

• request

  (WeightTransferInitRequest | dict) –

  Weight transfer initialization request with backend-specific info

Source code in vllm/entrypoints/rl/offline.py

def init_weight_transfer_engine(
    self, request: WeightTransferInitRequest | dict
) -> None:
    """Initialize weight transfer for RL training.

    Args:
        request: Weight transfer initialization request with backend-specific info

    """
    init_info_dict = (
        request["init_info"] if isinstance(request, dict) else request.init_info
    )

    self.llm_engine.collective_rpc(
        "init_weight_transfer_engine", kwargs={"init_info": init_info_dict}
    )

start_draft_weight_update()

Start a new weight update targeting the speculative draft model.

Source code in vllm/entrypoints/rl/offline.py

def start_draft_weight_update(self) -> None:
    """Start a new weight update targeting the speculative draft model."""
    self.llm_engine.collective_rpc("start_draft_weight_update")

start_weight_update()

Start a new weight update.

Source code in vllm/entrypoints/rl/offline.py

def start_weight_update(self) -> None:
    """Start a new weight update."""
    self.llm_engine.collective_rpc("start_weight_update")

update_weight_version(new_version)

Set the weight version without updating weights.

Source code in vllm/entrypoints/rl/offline.py

def update_weight_version(self, new_version: str) -> None:
    """Set the weight version without updating weights."""
    self.llm_engine.set_weight_version(new_version)

update_weights(request)

Update the weights of the model.

Parameters:

• request

  (WeightTransferUpdateRequest | dict) –

  Weight update request with backend-specific update info

Source code in vllm/entrypoints/rl/offline.py

def update_weights(self, request: WeightTransferUpdateRequest | dict) -> None:
    """Update the weights of the model.

    Args:
        request: Weight update request with backend-specific update info

    """
    update_info_dict = (
        request["update_info"] if isinstance(request, dict) else request.update_info
    )

    self.llm_engine.collective_rpc(
        "update_weights", kwargs={"update_info": update_info_dict}
    )