Safe Haskell	Safe-Inferred
Language	Haskell2010

Hpp.Tokens

Description

Tokenization breaks a String into pieces of whitespace, constants, symbols, and identifiers.

Synopsis

data Token s
- = Important s
- | Other s
detok :: Token s -> s
isImportant :: Token s -> Bool
notImportant :: Token s -> Bool
importants :: [Token s] -> [s]
trimUnimportant :: [Token s] -> [Token s]
detokenize :: Monoid s => [Token s] -> s
tokenize :: Stringy s => s -> [Token s]
newLine :: (Eq s, IsString s) => Token s -> Bool
skipLiteral :: Stringy s => s -> (s, s)

Documentation

data Token s Source #

Tokenization is words except the white space is tagged rather than discarded.

Constructors

Important s	Identifiers, symbols, and constants
Other s	White space, etc.

Instances

Instances details

Functor Token Source #
Instance details Defined in Hpp.Tokens Methods fmap :: (a -> b) -> Token a -> Token b # (<$) :: a -> Token b -> Token a #
Eq s => Eq (Token s) Source #
Instance details Defined in Hpp.Tokens Methods (==) :: Token s -> Token s -> Bool # (/=) :: Token s -> Token s -> Bool #
Ord s => Ord (Token s) Source #
Instance details Defined in Hpp.Tokens Methods compare :: Token s -> Token s -> Ordering # (<) :: Token s -> Token s -> Bool # (<=) :: Token s -> Token s -> Bool # (>) :: Token s -> Token s -> Bool # (>=) :: Token s -> Token s -> Bool # max :: Token s -> Token s -> Token s # min :: Token s -> Token s -> Token s #
Show s => Show (Token s) Source #
Instance details Defined in Hpp.Tokens Methods showsPrec :: Int -> Token s -> ShowS # show :: Token s -> String # showList :: [Token s] -> ShowS #

detok :: Token s -> s Source #

Extract the contents of a Token.

isImportant :: Token s -> Bool Source #

True if the given Token is Important; False otherwise.

notImportant :: Token s -> Bool Source #

True if the given Token is not Important; False otherwise.

importants :: [Token s] -> [s] Source #

Return the contents of only Important (non-space) tokens.

trimUnimportant :: [Token s] -> [Token s] Source #

Trim Other Tokens from both ends of a list of Tokens.

detokenize :: Monoid s => [Token s] -> s Source #

Collapse a sequence of Tokens back into a String. detokenize . tokenize == id.

tokenize :: Stringy s => s -> [Token s] Source #

Break an input String into a sequence of Tokens. Warning: This may not exactly correspond to your target language's definition of a valid identifier!

newLine :: (Eq s, IsString s) => Token s -> Bool Source #

Is a Token a newline character?

skipLiteral :: Stringy s => s -> (s, s) Source #

Skip over a string or character literal returning the literal and the remaining the input.