export type SyntaxToken = {
    type: string;
    start: number;
    end: number;
};
/**
 * Per-line token structure. Each line stores its own tokens with
 * line-relative start/end offsets (0 = start of line).
 */
export type TokenLine = SyntaxToken[];
/** Threshold below which we tokenize synchronously (characters). */
export declare const SYNC_TOKENIZE_THRESHOLD = 2000;
/**
 * Tokenizes a code string into per-line token arrays.
 *
 * Each line's tokens use line-relative offsets. Tokens matching the
 * language's default color type are omitted (~30% reduction).
 *
 * @param code - The source code string to tokenize
 * @param language - Language identifier (e.g. 'typescript', 'python')
 * @returns Array of token arrays, one per line
 */
export declare function tokenize(code: string, language: string): TokenLine[];
/**
 * Async tokenizer that yields to the main thread between batches
 * of lines (~800 chars per batch).
 */
export declare function tokenizeAsync(code: string, language: string, signal?: AbortSignal): Promise<TokenLine[]>;
/**
 * Streaming tokenizer with per-batch callback. Tokenizes lines in
 * batches and invokes `onBatch` after each so consumers can apply
 * highlights progressively.
 */
export declare function tokenizeStreaming(code: string, language: string, onBatch: (lines: TokenLine[], startLine: number) => void, signal?: AbortSignal): Promise<void>;
/**
 * Convert legacy flat tokens (absolute offsets) to per-line tokens
 * (line-relative offsets). For backwards compatibility with custom
 * tokenizers that return the old format.
 */
export declare function flatTokensToLines(tokens: {
    type: string;
    start: number;
    end: number;
}[], code: string): TokenLine[];
//# sourceMappingURL=tokenizer.d.ts.map