chatml

ChatML to canonical string parser

Attributes

STATE_INITIAL module-attribute

STATE_INITIAL = 'INITIAL'

STATE_MESSAGE module-attribute

STATE_MESSAGE = 'message'

STATE_START module-attribute

STATE_START = 'start'

states module-attribute

states = (
    (STATE_START, "exclusive"),
    (STATE_MESSAGE, "exclusive"),
)

t_WHITESPACE module-attribute

t_WHITESPACE = '[ \\t]+'

t_message_error module-attribute

t_message_error = t_error

t_start_WHITESPACE module-attribute

t_start_WHITESPACE = '[ \\t]+'

t_start_error module-attribute

t_start_error = t_error

tokens module-attribute

tokens = (
    "START",
    "ROLE",
    "NAME",
    "MESSAGE",
    "END",
    "NEWLINE",
    "WHITESPACE",
)

Classes

ChatML

ChatML(env: Environment)

Bases: BaseCanonical

Attributes

name property

name: str

Functions

to_canonical

to_canonical(data: str, **kwargs) -> CanonicalChat

Functions

t_NEWLINE

t_NEWLINE(t: LexToken)

\n

t_START

t_START(t: LexToken)

<|im_start|>

t_error

t_error(t: LexToken)

t_message_END

t_message_END(t: LexToken)

<|im_end|>

t_message_MESSAGE

t_message_MESSAGE(t: LexToken)

[\s\S]+?(?=<|im_end|>)

t_start_NAME

t_start_NAME(t: LexToken)

name[ \t]=[ \t][a-zA-Z0-9]+

t_start_NEWLINE

t_start_NEWLINE(t: LexToken)

\n

t_start_ROLE

t_start_ROLE(t: LexToken)

(system|user|assistant|tool)