Tokenizer in QB64

[bplus]

Yes I mean Declare 's Sub and or Function, it's OK but just another place that can screw you up.

Code: (Select All)

'tokenizer in QB (fb) by Aurel
Const tkNULL = 0, tkPLUS = 1, tkMINUS = 2, tkMULTI = 3, tkDIVIDE = 4
Const tkCOLON = 5, tkCOMMA = 6, tkLPAREN = 7, tkRPAREN = 8, tkLBRACKET = 9, tkRBRACKET = 10
Const tkIDENT = 11, tkNUMBER = 12, tkQSTRING = 13, tkCOMMAND = 14, tkEOL = 15
Const tkEQUAL = 16, tkMORE = 17, tkLESS = 18, tkAND = 19, tkOR = 20, tkNOT = 21
Const tkHASH = 22, tkSSTR = 23, tkMOD = 24, tkSEMI = 25, tkDOT = 26, tkLBRACE = 27, tkRBRACE = 28
Const tkQUEST = 29, tkMONKEY = 30, tkBACKSLAH = 31, tkPOWUP = 32, tkAPOSTR = 33, tkTILDA = 34

Dim Shared tokList(1024) As String 'token array
Dim Shared typList(1024) As Integer 'token type array
Dim Shared p As Long: p = 1
Dim Shared start As Long: start = 1
Dim Shared tp As Long
Dim Shared tn As Long
Dim Shared n As Long
Dim Shared ltp As Long: lpt = 1
Dim Shared nTokens As Long 'nTokens -> number of tokens
Dim Shared lineCount As Integer
Dim Shared Lpar As Integer
Dim Shared Rpar As Integer
Dim Shared Lbrk As Integer
Dim Shared Rbrk As Integer
Dim Shared tokerr As Integer
Dim Shared codeLen As Integer
Dim Shared code As String
Dim Shared chs As String
Dim Shared tch As String
Dim Shared tk As String
Dim Shared crlf As String
Dim Shared bf As String
Dim Shared ntk As String
crlf = Chr$(13) + Chr$(10)
'test string .......................................
Dim test As String: test = "func tokenizer in QB64"
'...................................................

'call fn tokenizer()
temp& = tokenizer&(test)


Do

Loop Until _KeyDown(27)

' *** MAIN TOKENIZER FUNCTION ***
Function tokenizer& (src As String)
    Print "tokenizer run:" + src
    lineCount = 0: ltp = start: nTokens = 0

    tokenizer& = 0
End Function

Can't use As Type for Function definitions need to use Type suffixes on the Function name.