# coding: iso-8859-1
#
# script(ed/able/ing/ful) code compiler
# (c)2006 Tiago Rezende
# Distributed freely.
#
# C/C++ Macro Preprocessor
#
import os
import re
from base import *
from fileclasses import *
def mapStrings(line, stringunendedinlastline):
# utility function to search for all strings in the current line
ignorenext = False
if stringunendedinlastline:
strlist = [0]
instring = True
else:
strlist = []
instring = False
for pos in xrange(len(line)):
c = line[pos]
if ignorenext:
ignorenext=False
elif c == '\\':
ignorenext=True
elif c == '"':
if instring:
strlist[0][1] = pos
instring = False
else:
strlist = [pos] + strlist
instring = True
if instring:
unended = True
else:
unended = False
return (strlist,unended)
def unendedStrings(line, stringunendedinlastline):
# utility function to search for all strings in the current line,
# and tell if a given line has an unended string
ignorenext = False
if stringunendedinlastline:
instring = True
else:
instring = False
for pos in xrange(len(line)):
c = line[pos]
if ignorenext:
ignorenext=False
elif c == '\\':
ignorenext=True
elif c == '"':
if instring:
instring = False
else:
instring = True
if instring:
return True
return False
def insideString(line, start, stringunendedinlastline):
# utility function to find if a certain index is inside a C-string
ignorenext = False
if stringunendedinlastline:
strlist = [0]
instring = True
else:
strlist = []
instring = False
for pos in xrange(len(line)):
c = line[pos]
if ignorenext:
ignorenext=False
elif c == '\\':
ignorenext=True
elif c == '"':
if instring:
strlist[0][1] = pos
instring = False
else:
strlist = [pos] + strlist
instring = True
if instring:
# don't assume a given line has a complete string
strlist[0][1]=len(line)
for x in strlist:
if start > x[0] and start < x[1]:
return True
return False
class LineEndingException(Exception):
pass
def findCloseString(line, start):
# find the end of a C-style string, considering special characters and new-lines.
char = line[start]
pos = start
while char!='"':
if pos>=len(line):
raise LineEndingException()
elif char=='\\':
pos = pos+1
pos = pos+1
char = line[pos]
return pos
def findCloseParen(line, start):
# find the matching parenthesis of a given expression, starting from
# the character right after the opening parenthesis, and going recursively
# until it either finds the closing one or throws up an exception (to which
# the parser responds by adding another line)
char = line[start]
pos = start
while char!=')':
if pos>=len(line):
raise LineEndingException()
if char=='/':
if line[pos+1]=='/':
pos=findNewLine(line,pos+2)
if line[pos+1]=='*':
pos=findCloseCStyleComment(line,pos+2)
elif char=='(':
pos = findCloseParen(line,pos+1)
elif char=='"':
pos = findCloseString(line,pos+1)
pos = pos+1
char = line[pos]
return pos
def findCloseCStyleComment(line, start):
# find the end of a C-style comment. if the end of the line is found before that,
# throw an exception so the caller can handle it by calling the function again
# for the next line.
char = line[start]
pos = start
while pos<len(line):
if char=='*':
if line[pos+1]=='/':
return pos+1
pos = pos+1
char = line[pos]
raise LineEndingException()
def findNewLine(line, start):
# find a new-line character.
char = line[start]
pos = start
while pos<len(line):
if char=='\n':
return pos
pos = pos+1
char = line[pos]
raise LineEndingException()
def extractDefineArguments(line, start):
# extracts the arguments from a matching define, returning a list of them.
# Start from right after the first parenthesis
char = line[start]
pos = start
lastargpos = pos
args = []
while pos<len(line):
if char==')':
return args
if char==',':
args = args + [line[lastargpos:pos-1]]
lastargpos=pos+1
if char=='(':
pos = findCloseParen(line,pos+1)
pos = pos + 1
char = line[pos]
raise LineEndingException()
class CPreprocessorDefineArgumentsError(Exception):
def __init__(self, string, errval, relpos):
self.string = string
self.value = errval
self.pos = relpos
def extractDefineNameAndArguments(line, pos):
# extracts the name and the arguments of a #define, putting them on a list
insideparens = False
token = ''
retval = []
argcount = -1
for char in line:
if char=='(':
if not insideparens:
insideparens = True
retval.append(token)
token = ''
argcount = 0
else:
raise CPreprocessorDefineArgumentsError("invalid parenthesis opening", 201, pos)
elif char==')':
if not insideparens:
raise CPreprocessorDefineArgumentsError("invalid parenthesis closing", 202, pos)
else:
if len(token) != 0:
retval.append(token)
return (argcount, retval, pos+1)
elif char==',':
retval.append(token)
argcount = argcount+1
elif char==' ' or char=="\t":
pass
else:
if len(token)==0:
if not char.isalpha() or char!='_':
raise CPreprocessorDefineArgumentsError("invalid character at beginning of token", 200, pos)
elif not char.isalnum() or char!='_':
raise CPreprocessorDefineArgumentsError("invalid character in middle of token", 201, pos)
token = token + char
pos = pos+1
class CPreprocessorUnrecoverableError(Exception):
def __init__(self, string, errorvalue):
self.string = string
self.errorvalue = errorvalue
def getValue(self):
return self.errorvalue
class CPreprocessor(BaseModule):
def __init__(self, infile, filename, options):
# add the current directory as a source of includes
self.dirs = ['.']
# add any dirs that are in the options
# get the INCLUDE environment variable, split it (by splitting on the ; ) and includes the individual directories with the other includes
includedirs = os.environ['INCLUDE']
if includedirs:
self.dirs + re.split(r';', includedirs)
# set a stack (as a list of NamedLinedFiles) with the input filename and file 'handle'
self.file = [NamedLinedFile(infile, filename)]
# set the defines dictionary
# defines dictionary description:
# key is the define declaration, first item in the list is the number of arguments
# then a list of strings and ints, the ints referring to the arguments to be substituted
# ex.: #define MAX(a,b) ((a>b)?a:b)
# parses to:
# self.defines['MAX']=[2, ['({',1,'>',2,')?',2,':',1,')'], filename.c, 10]
# defines with no parentheses are stored with number of params -1
self.defines = {'__SCC__':[-1, ['1'], "internal compiler defines", -1]}
# set the line output list
self.lines = []
self.unendedstring = False
# #if/#ifdef/#ifndef/#endif inverse stack
self.ifstack = []
self.verboselevel = 0
def addIncludeDir(self, _dir):
# append the directory to the directory list
self.dirs.append(_dir)
self.addMessage("appended directory " + _dir, 10)
#def addDefine(self, define, definition=[-1,'']):
# # add (or substitute) define to the defines directory
# self.defines[define] = definition
def includeFile(self, filename):
for _dir in self.dirs:
# try to find in the directory list where the file is
infile = open(os.path.join(_dir,filename),'r')
if infile:
# if found set the current file to it and return
self.file = [NamedLinedFile(infile, os.path.join(_dir,filename))] + self.file
self.addMessage("found include file " + self.file[0].getFileName(), 10)
return
# else print the error and append it to the list of errors (and stop parsing by throwing an error)
err = ""+ self.file[0].getFileName()+"<"+self.file[0].getLineNum()+">: could not include " + filename + ", file not found"
self.printError(err, -101)
raise CPreprocessorUnrecoverableError(err, -101)
def parseLine(self):
# read a line
line = self.file[0].readline()
linenum = self.file[0].getLineNum()
filename = self.file[0].getFileName()
# set __FILE__ to the current file
self.defines["__FILE__"] = [-1,linenum]
# set __LINE__ to the current line
self.defines["__LINE__"] = [-1,filename]
self.addMessage("file "+filename+" line "+linenum, 13)
# apply defines, taking care not to trip into any string, and
# assuring no define is forgotten in case it is in the result
# of another define
foundparseable = True
while foundparseable:
# if no define is found, exit the loop
foundparseable = False
for define in self.defines.keys():
found = line.find(define)
while found>=0:
self.addMessage("found define " + define + " at <"+linenum+","+found+">", 11)
if not insideString(line, found, self.unendedstring):
# found a define
foundparseable = True
# substitute the string with the defined
if self.defines[define][0]<0:
# if there's no argument list, just substitute the define term with
# the defined string
line = line[:found]+self.defines[define][1]+line[found+len(define):]
else:
appliedline = line[:found]
arguments = extractDefineArguments(line,found+len(define)+1)
if len(arguments) != self.defines[define][0]:
error = "" + self.file[0].getFileName() + "<" + self.file[0].getLineNum()+">: " + define +" argument count doesn't match specification at " + self.defines[define][2] + " at line " + self.defines[define][2]
self.addError(error, -50)
for element in self.defines[define][1]:
if isinstance(element, (int)):
if len(arguments) < element:
error = "" + self.file[0].getFileName() + " <" + self.file[0].getLineNum()+">: couldn't substitute argument #"+ element+" from define " + define +" specified at " + self.defines[define][2] + " at line " + self.defines[define][2]
self.addError(error, -102)
raise CPreprocessorUnrecoverableError(error,-102)
appliedline = appliedline + arguments[element]
else:
appliedline = appliedline + element
# try finding it once again
found = line.find(define, found+len(define))
# parse //... comments and remove them
# add later a flag to throw an error if not under C99(?) standard / GCC compatibility mode
found = line.find('//')
while found >= 0:
if not insideString(line, found, self.unendedstring):
#get only the first part of the string
line = line[:found]
self.addMessage("found C++ style (line-terminating) comment at <"+linenum+","+found+">",12)
else:
#continue trying to find it
found = line.find('//', found+2)
# parse /* ... */ comments and remove them, removing the characters
# (and maybe lines) upto the end of the comments
found = line.find('/*')
while found >= 0:
if not insideString(line, found, self.unendedstring):
self.addMessage("found C style comment at <"+linenum+","+found+">",12)
line2 = line[found:]
found2 = line2.find('*/')
while found2 < 0:
try:
line2 = line2 + self.file[0].readline()
except:
error = "" + self.file[0].getFileName() + "<" + self.file[0].getLineNum()+">: unexpected end-of-file; expected comment ending"
self.addError(error,-100)
raise CPreprocessorUnrecoverableError(error,-100)
found2 = line2.find('*/')
line = line[:found] + line2[found2:]
# try to find another one
found = line.find('/*', found+1)
# sees if the line ends with '\' - if so we pick the next line
while line.endswith('\\'):
# take it off and add the next line
line = line[:len(line)-2]+self.file[0].readline()
self.addMessage("found special character \"\\\" at <"+linenum+">, joining the working line to the next",12)
# parse #defines, #if, #pragmas, etc...
if line[0]=='#':
# take the '#' off and strip spaces
read = line[1:].strip()
# get a lower case version of the stripped string
low = read.lower()
# if it finds 'define' in the first characters
if low.find('define ')==0:
# extract the '#define' bit
line = read[7:].strip()
# get the define and put it in the dictionary
try:
argcount, arguments, pos = extractDefineNameAndArguments(line, 0)
line = line[pos:]
self.addMessage("found #define "+arguments[0]+" at <"+linenum+">",12)
self.defines[arguments[0]] = [argcount,[],self.file[0].getFileName(),self.file[0].getLineNum()]
except CPreprocessorDefineArgumentsError, e:
error = "" + self.file[0].getFileName() + "<" + self.file[0].getLineNum()+">: " + e.string
self.addError(error,e.value)
elif low.find('undef ')==0:
define = read[6:].split()
if isCIdentifierFormat(define):
if self.defines[define]:
del self.defines[define]
self.addMessage("found #undef "+define+" at <"+linenum+">",12)
else:
error = "" + self.file[0].getFileName() + "<" + self.file[0].getLineNum()+">: "
self.addError(error, -111)
# if it finds 'ifdef'
elif low.find('ifdef ')==0:
# see if the word is in the dictionary, if it is, continue.
# if not, stop outputting until the next #elif, #else or #endif.
tokens = read[6:].split()
if(len(tokens)!=1):
error = "" + self.file[0].getFileName() + "<" + self.file[0].getLineNum()+">: macro names must be identifiers"
self.addError(error,-111)
define = tokens[0]
if not isCIdentifierFormat(define):
error = "" + self.file[0].getFileName() + "<" + self.file[0].getLineNum()+">: macro names must be identifiers"
self.addError(error,-110)
if self.defines[define]:
self.ifstack = [True] + self.ifstack
self.addMessage("found #ifdef " + define + " at <"+linenum+">, evaluated to True",12)
else:
self.ifstack = [False] + self.ifstack
self.addMessage("found #ifdef " + define + " at <"+linenum+">, evaluated to False",12)
elif low.find('ifndef ')==0:
# see if the word is in the dictionary, if it is, stop outputting
# until the next #elif, #else or #endif. if not, continue.
tokens = read[7:].split()
if(len(tokens)!=1):
error = "" + self.file[0].getFileName() + "<" + self.file[0].getLineNum()+">: macro names must be identifiers"
self.addError(error,-111)
define = tokens[0]
if not isCIdentifierFormat(define):
error = "" + self.file[0].getFileName() + "<" + self.file[0].getLineNum()+">: macro names must be identifiers"
self.addError(error,-110)
if self.defines[define]:
self.ifstack = [False] + self.ifstack
self.addMessage("found #ifdef " + define + " at <"+linenum+">, evaluated to False",12)
else:
self.ifstack = [True] + self.ifstack
self.addMessage("found #ifdef " + define + " at <"+linenum+">, evaluated to True",12)
elif low.find('if')==0:
# see if the next word, number or expression is not 0 or undefined, if it is, stop output until the matching #endif
print '#if found:'
print line
elif low.find['elif ']==0:
# close matching if/ifdef/ifndef, open another if
print '#elif found:'
print line
elif low.find['endif']==0:
if read[5:].strip()!='':
warning = "" + self.file[0].getFileName() + "<" + self.file[0].getLineNum()+">: extra tokens at the end of #endif"
self.addWarning(line, -56, 0)
else:
del self.ifstack[0]
elif low.find('error ')==0:
self.addError(line, -199)
elif low.find('warning ')==0:
self.addWarning(line, -198, 0)
# if it finds 'pragma'
elif low.find('pragma ')==0:
# pass arguments to the compiler/assembler/linker
print '#pragma found:'
print line
else:
# add an error signing an unrecognized keyword
error = ""+filename+" <"+ linenum +">: unrecognized preprocessor keyword " + read.split()[0]
self.addError(error, -55)
else:
# return the resulting line (along with filename and line number)
return line, [filename, linenum]
def parseAll(self):
# while no errors logged
while 1:
# try parsing a single line
try:
line = self.parseLine()
self.lines.append(line)
except FileEndingException:
# if file ends, get another file
if self.files[1]:
self.files[0].closeFile()
del self.files[0]
# return to the loop
pass
else:
# break the loop and return it all
self.files[0].closeFile()
break
return self.output()
def output(self):
# return a tuple with the lines, warnings and errors
return self.lines
