11-21-2023, 04:58 PM
(11-21-2023, 04:13 PM)aurel Wrote: hello
I finished translation of my tokenizer from o2 to QB64
whole code compile fine
but when i run program i get ILLEGAL FUNCTION CALL
and the weird thing it is on LINE 112
BUT there is no any function call on line 112 ...anyone ?
Code: (Select All)'tokenizer in QB64pe (fb) by Aurel
declare function tokenizer&( src as string)
'declare function run_tokenizer(inputCode as string) as integer
const tkNULL=0, tkPLUS=1, tkMINUS=2, tkMULTI=3, tkDIVIDE=4
const tkCOLON=5, tkCOMMA=6, tkLPAREN=7, tkRPAREN=8, tkLBRACKET=9, tkRBRACKET=10
const tkIDENT = 11 , tkNUMBER = 12 , tkQSTRING = 13, tkCOMMAND =14 ,tkEOL = 15
const tkEQUAL = 16, tkMORE = 17, tkLESS = 18, tkAND = 19, tkOR = 20, tkNOT = 21
const tkHASH=22 , tkSSTR=23, tkMOD=24 , tkSEMI=25, tkDOT=26, tkLBRACE=27, tkRBRACE=28
const tkQUEST=29, tkMONKEY=30 , tkBACKSLAH=31, tkPOWUP=32 ,tkAPOSTR=33 , tkTILDA=34
Dim shared tokList(1024) As string 'token array
Dim shared typList(1024) As integer 'token type array
Dim shared p As Long : p=1
Dim shared start as Long : start = 1
Dim shared tp as long
Dim shared tn as long
Dim shared n as long
Dim shared ltp as long : lpt = 1
Dim shared nTokens As long 'nTokens -> number of tokens
Dim shared lineCount As integer
Dim shared Lpar as integer
Dim shared Rpar as integer
Dim shared Lbrk as integer
Dim shared Rbrk as integer
Dim shared tokerr as integer
Dim shared codeLen as integer
Dim shared code As String
Dim shared chs As String
Dim shared tch As String
Dim shared tk As String
Dim shared crlf As String
Dim shared bf As String
Dim shared ntk As String
Dim shared ch As String
crlf = chr$(13) + chr$(10)
'test string .......................................................
Dim test as string : test = "func tokenizer in QB64"
'...................................................................
tn = tokenizer&(test)
'print result on screen...
PRINT "Number of tokens: " + str$(tn)
PRINT "Number of lines: " + str$(lineCount)
nTokens = tn
' *** MAIN TOKENIZER FUNCTION ***
FUNCTION tokenizer& (src as string )
print "tokenizer run:" + src
lineCount=0:ltp=start : nTokens = 0
'Main Tokenizer Loop.....................................
WHILE p <= len(src)
'------------------------------------
ch = Mid$(src,p,1) ' get char
'------------------------------------
If Asc(ch)=32 Then p=p+1 ' skip blank space[ ]
If Asc(ch)=9 Then p=p+1 ' skip TAB [ ]
If Asc(ch)=13 Then p=p+1 ' skip CR -> goto Again -> NewLine
if asc(ch)=39 Then ' skip comment line[ ' ]
while asc(ch) <> 10
p=p+1 : ch = mid$(src,p,1)
if asc(ch)= 10 OR asc(ch) = 0 THEN Exit While
wend
lineCount=lineCount+1 : tp=tp+1 : tokList(tp)="EOL" : typList(tp)= tkEOL : tk="": ch="" ' add EOL on comment end
p=p+1 : goto endLoop ' jump to LABEL -> end of loop
end if
If asc(ch)=10 Then ' EOL
if Lpar > Rpar Then
tokerr=3 : goto tokExit
end if ' if Rparen ((...)
if Lpar < Rpar Then
tokerr=4 : goto tokExit
end if 'if Lparen (...))
if Lbrk > Rbrk Then
tokerr=5 : goto tokExit
end if ' if Lbracket [..
if Lbrk < Rbrk Then
tokerr=6 : goto tokExit
end if ' if Rbracket ...]
lineCount=lineCount+1 : tp=tp+1 : tokList(tp)="EOL" :typList(tp)= tkEOL: tk="": ch="" : p=p+1
End if
'--------------------------------------------------------
If asc(ch)=34 Then ' if char is QUOTE "
p=p+1 : ch = Mid$(src,p,1) : tk=ch : p=p+1 ' skip quote :add ch TO tk buffer: p+1
while asc(ch) <> 34
ch = Mid$(src,p,1) : if asc(ch)= 34 then exit while
tk=tk+ch : p=p+1
IF ch = chr$(10) Then
tokerr = 2: goto tokExit
END IF
wend
tp=tp+1 : tokList(tp)= tk :typList(tp)= tkQSTRING: tk="":ch="": p=p+1 ' add quoted string to token list
End if
'-------------------------------------------------------
If (asc(ch)>96 and asc(ch)<123) or (asc(ch)>64 and asc(ch)<91) or asc(ch)=95 Then ' [a-z,A-Z_]
while (asc(ch)>96 and asc(ch)<123) or (asc(ch)>64 and asc(ch)<91) or (asc(ch)>47 and asc(ch)<58) or asc(ch)=95 ' [a-z,A-Z,0-9_]
tk=tk+ch : p=p+1 : ch = mid$(src,p,1)
wend
' ' add token ,add token type/IDENT:{VAR/COMMAND}
tp=tp+1 : tokList(tp) = tk :typList(tp)= tkIDENT: tk="":ch=""
End If
'--------------------------------------------------------------
If (asc(ch)>47 and asc(ch)<58) Then ' [0-9.]
while (asc(ch)>47 AND asc(ch)<58) OR asc(ch)=46 ' [0-9[0.0]]*
tk=tk+ch :p=p+1 : ch = mid$(src,p,1)
wend
' add token ,add token type/NUMBER
tp=tp+1 : tokList(tp) = tk : typList(tp)= tkNUMBER: tk="":ch=""
End if
'-------------------------------------------------------------------------
If asc(ch)=43 Then
tp=tp+1 : tokList(tp) = ch :typList(tp)= tkPLUS: ch="" : p=p+1
End If
If asc(ch)=45 Then
tp=tp+1 : tokList(tp) = ch :typList(tp)= tkMINUS: ch="" : p=p+1
End If
If asc(ch)=42 Then
tp=tp+1 : tokList(tp) = ch :typList(tp)= tkMULTI: ch="" : p=p+1
End If
If asc(ch)=47 Then
tp=tp+1 : tokList(tp) = ch :typList(tp)= tkDIVIDE: ch="" : p=p+1
End If
If asc(ch)=40 then tp=tp+1 : tokList(tp) = ch :typList(tp)= tkLPAREN: ch="" : p=p+1 : Lpar=Lpar+1
If Asc(ch)=41 then tp=tp+1 : tokList(tp) = ch :typList(tp)= tkRPAREN: ch="" : p=p+1 : Rpar=Rpar+1 ' ) Rparen
If Asc(ch)=44 then tp=tp+1 : tokList(tp) = ch :typList(tp)= tkCOMMA: ch="" : p=p+1 ' , comma
If Asc(ch)=58 then tp=tp+1 : tokList(tp) = ch :typList(tp)= tkCOLON: ch="" : p=p+1 ' : colon
If Asc(ch)=59 then tp=tp+1 : tokList(tp) = ch :typList(tp)= tkSEMI : ch="" : p=p+1 ' ; semi_colon
If Asc(ch)=60 then tp=tp+1 : tokList(tp) = ch :typList(tp)= tkLESS: ch="" : p=p+1 ' < less
If Asc(ch)=61 then tp=tp+1 : tokList(tp) = ch :typList(tp)= tkEQUAL: ch="" : p=p+1 ' = equal
If Asc(ch)=62 then tp=tp+1 : tokList(tp) = ch :typList(tp)= tkMORE: ch="" : p=p+1 ' > more(greater)
If Asc(ch)=63 then tp=tp+1 : tokList(tp) = ch :typList(tp)= tkQUEST: ch="" : p=p+1 ' > questMark ?
If Asc(ch)=64 then tp=tp+1 : tokList(tp) = ch :typList(tp)= tkMONKEY: ch="" : p=p+1 ' > at(monkey) @
If Asc(ch)=91 then tp=tp+1 : tokList(tp) = ch :typList(tp)= tkLBRACKET:ch="" : p=p+1 : Lbrk=Lbrk+1 ' ( Lbracket
If Asc(ch)=92 then tp=tp+1 : tokList(tp) = ch :typList(tp)= tkBACKSLAH:ch="" : p=p+1 : : ' \ backSlash
If Asc(ch)=93 then tp=tp+1 : tokList(tp) = ch :typList(tp)= tkRBRACKET:ch="" : p=p+1 : Rbrk=Rbrk+1 ' ) Rbracket
If Asc(ch)=94 then tp=tp+1 : tokList(tp) = ch :typList(tp)= tkPOWUP: ch="" : p=p+1 ' ^ power up
If Asc(ch)=96 then tp=tp+1 : tokList(tp) = ch :typList(tp)= tkAPOSTR: ch="" : p=p+1 ' ` apoStrophe
If Asc(ch)=38 then tp=tp+1 : tokList(tp) = ch :typList(tp)= tkAND: ch="" : p=p+1 ' & AND
If Asc(ch)=124 then tp=tp+1 : tokList(tp) = ch :typList(tp)= tkOR: ch="" : p=p+1 ' | OR
If Asc(ch)=33 then tp=tp+1 : tokList(tp) = ch :typList(tp)= tkNOT: ch="" : p=p+1 ' ! NOT
If Asc(ch)=35 then tp=tp+1 : tokList(tp) = ch :typList(tp)= tkHASH: ch="" : p=p+1 ' # hash
If Asc(ch)=36 then tp=tp+1 : tokList(tp) = ch :typList(tp)= tkSSTR: ch="" : p=p+1 ' $ $TRING
If Asc(ch)=37 then tp=tp+1 : tokList(tp) = ch :typList(tp)= tkMOD : ch="" : p=p+1 ' % percent/MOD
If Asc(ch)=46 then tp=tp+1 : tokList(tp) = ch :typList(tp)= tkDOT : ch="" : p=p+1 ' . dot/point
If Asc(ch)=123 then tp=tp+1 : tokList(tp) = ch :typList(tp)= tkLBRACE :ch="" : p=p+1 ' { LBrace
If Asc(ch)=125 then tp=tp+1 : tokList(tp) = ch :typList(tp)= tkRBRACE :ch="" : p=p+1 ' } RBrace
If Asc(ch)=126 then tp=tp+1 : tokList(tp) = ch :typList(tp)= tkTILDA :ch="" : p=p+1 ' ~ tilda
IF ASC(ch)>126 then tokerr = 1 : goto tokExit
IF ASC(ch)<28 then tokerr = 1
IF ASC(ch)=0 then tokerr = 0
IF ASC(ch)=9 then tokerr = 0
IF ASC(ch)=10 then tokerr = 0
IF ASC(ch)=13 then tokerr = 0
IF tokerr = 1 then goto tokExit
endLoop:
WEND
tokenizer = tp
If tp <> 0 then EXIT FUNCTION
tokExit:
IF tokerr > 0 THEN
if tokerr = 1 then PRINT "Unknown token!-[ " + ch +" ] at LINE: " + str$(lineCount)
if tokerr = 2 then PRINT "Unclosed Quote!- at LINE: " + str$(lineCount)
if tokerr = 3 then PRINT "Missing right paren! ((...)- at LINE: " + str$(lineCount)
if tokerr = 4 then PRINT "Missing left paren! (...))- at LINE: " + str$(lineCount)
if tokerr = 5 then PRINT "Missing right bracket!- at LINE: " + str$(lineCount)
if tokerr = 6 then PRINT "Missing left bracket!- at LINE: " + str$(lineCount)
tokenizer& = 0 : EXIT FUNCTION
END IF
tokenizer = 0
End Function
'tokout& = tokenizer&(test) ' if is called after function then cause error
Line 112:
IF (ASC(ch) > 47 AND ASC(ch) < 58) THEN ' [0-9.]
ASC is a function call and it throws the error if it gets an empty string as input, so you should check if your variable "ch" is non-empty before passing it to ASC.
GuiTools, Blankers & other Projects:
https://qb64phoenix.com/forum/forumdisplay.php?fid=32
Libraries & useful Functions:
https://qb64phoenix.com/forum/forumdisplay.php?fid=23
https://qb64phoenix.com/forum/forumdisplay.php?fid=32
Libraries & useful Functions:
https://qb64phoenix.com/forum/forumdisplay.php?fid=23