From a5abaf8d69e618fbdbd195a82052a5bdb9daa08c Mon Sep 17 00:00:00 2001 From: Laurent Pinson Date: Fri, 17 Dec 2021 12:38:29 +0100 Subject: [PATCH 01/13] rewrote code in pythonic ways --- pypreprocessor/__init__.py | 91 ++++++++++++++------------------------ 1 file changed, 33 insertions(+), 58 deletions(-) diff --git a/pypreprocessor/__init__.py b/pypreprocessor/__init__.py index b0b2a07..b292a68 100644 --- a/pypreprocessor/__init__.py +++ b/pypreprocessor/__init__.py @@ -27,13 +27,9 @@ def __init__(self, inFile=sys.argv[0], outFile='', defines=[], \ self.save = save self.readEncoding = sys.stdin.encoding self.writeEncoding = sys.stdout.encoding + # private variables - self.__linenum = 0 - self.__excludeblock = False - self.__ifblocks = [] - self.__ifconditions = [] - self.__evalsquelch = True - self.__outputBuffer = '' + self.__reset_internal() def check_deprecation(self): def deprecation(message): @@ -86,17 +82,11 @@ def undefine(self, define): # search: if define is defined def search_defines(self, define): - if define in self.defines: - return True - else: - return False + return define in self.defines: #returning: validness of #ifdef #else block def __if(self): - value = bool(self.__ifblocks) - for ib in self.__ifblocks: - value *= ib #* represents and: value = value and ib - return not value #not: because True means removing + return not (self.__ifblocks and all(self.__ifblocks)) # evaluate def lexer(self, line): @@ -208,9 +198,6 @@ def lexer(self, line): for i in range(0, number): self.__ifblocks.pop(-1) self.__ifcondition = self.__ifconditions.pop(-1) - elif len(self.__ifconditions) == number: - self.__ifblocks = [] - self.__ifconditions = [] else: print('Warning try to remove more blocks than present', \ self.input, self.__linenum) @@ -218,58 +205,47 @@ def lexer(self, line): self.__ifconditions = [] return False, True else: #No directive --> execute - # process the excludeblock - if self.__excludeblock is True: - return True, False - # process the ifblock - elif self.__ifblocks: # is True: - return self.__if(), False - #here can add other stuff for deleting comnments eg - else: - return False, False + return self.__excludeblock or self.__ifblocks, False # error handling def exit_error(self, directive): print('File: "' + self.input + '", line ' + str(self.__linenum)) print('SyntaxError: Invalid ' + directive + ' directive') sys.exit(1) + def rewrite_traceback(self): trace = traceback.format_exc().splitlines() - index = 0 + trace[-2]=trace[-2].replace("", self.input)) for line in trace: - if index == (len(trace) - 2): - print(line.replace("", self.input)) - else: - print(line) - index += 1 + print(line) + # parsing/processing def parse(self): self.__reset_internal() self.check_deprecation() # open the input file - input_file = io.open(os.path.join(self.input), 'r', encoding=self.readEncoding) try: - # process the input file - for line in input_file: - self.__linenum += 1 - # to squelch or not to squelch - squelch, metaData = self.lexer(line) - # process and output - if self.removeMeta is True: - if metaData is True or squelch is True: + with io.open(os.path.join(self.input), 'r', encoding=self.readEncoding) as input_file: + # process the input file + for line in input_file: + self.__linenum += 1 + # to squelch or not to squelch + squelch, metaData = self.lexer(line) + # process and output + if self.removeMeta is True: + if metaData is True or squelch is True: + continue + if squelch is True: + if metaData: + self.__outputBuffer += self.escape + line + else: + self.__outputBuffer += self.escape[0] + line + continue + if squelch is False: + self.__outputBuffer += line continue - if squelch is True: - if metaData: - self.__outputBuffer += self.escape + line - else: - self.__outputBuffer += self.escape[0] + line - continue - if squelch is False: - self.__outputBuffer += line - continue finally: - input_file.close() #Warnings for unclosed #ifdef blocks if self.__ifblocks: print('Warning: Number of unclosed Ifdefblocks: ', len(self.__ifblocks)) @@ -295,12 +271,12 @@ def post_process(self): # set file name if self.output == '': self.output = self.input[0:-len(self.input.split('.')[-1])-1]+'_out.'+self.input.split('.')[-1] - # open file for output - output_file = io.open(self.output, 'w', encoding=self.writeEncoding) + # write post-processed code to file - output_file.write(self.__outputBuffer) + with io.open(self.output, 'w', encoding=self.writeEncoding) as output_file: + output_file.write(self.__outputBuffer) finally: - output_file.close() + pass if self.run: # if this module is loaded as a library override the import @@ -335,9 +311,8 @@ def override_import(self): # postprocessor - on-the-fly execution def on_the_fly(self): try: - f = io.open(self.output, "r", encoding=self.readEncoding) - exec(f.read()) - f.close() + with io.open(self.output, "r", encoding=self.readEncoding) as f: + exec(f.read()) except: self.rewrite_traceback() From 7cabdf0787d40990a5104fe916f84cf1e65cb8e1 Mon Sep 17 00:00:00 2001 From: Laurent Pinson Date: Fri, 17 Dec 2021 13:16:50 +0100 Subject: [PATCH 02/13] changed type of Preprocessor::defines to Dict + add support for #if --- pypreprocessor/__init__.py | 53 ++++++++++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 11 deletions(-) diff --git a/pypreprocessor/__init__.py b/pypreprocessor/__init__.py index b292a68..eded5b1 100644 --- a/pypreprocessor/__init__.py +++ b/pypreprocessor/__init__.py @@ -10,11 +10,17 @@ import traceback import imp import io +import collections + + class preprocessor: - def __init__(self, inFile=sys.argv[0], outFile='', defines=[], \ + def __init__(self, inFile=sys.argv[0], outFile='', defines={}, \ removeMeta=False, escapeChar=None, mode=None, escape='#', \ run=True, resume=False, save=True): # public variables + #support for <=0.7.7 + if isinstance(defines, collections.Sequence): + defines = {x:None for x in defines} self.defines = defines self.input = inFile self.output = outFile @@ -37,10 +43,12 @@ def deprecation(message): warnings.simplefilter('always', DeprecationWarning) warnings.warn(message, DeprecationWarning) warnings.simplefilter('default', DeprecationWarning) + if self.escapeChar != None: deprecation("'pypreprocessor.escapeChar' is deprecated. Use 'escape' instead.") if self.escape == '#': self.escape = self.escapeChar + if self.mode != None: msg = "'pypreprocessor.mode' is deprecated. Use 'run/resume/save' options instead." if self.run != True or self.resume != False or self.save != True: @@ -66,23 +74,33 @@ def deprecation(message): def __reset_internal(self): self.__linenum = 0 self.__excludeblock = False - self.__ifblocks = [] - self.__ifconditions = [] + self.__ifblocks = [] # contains the evaluated if conditions + self.__ifconditions = [] # contains the if conditions self.__evalsquelch = True self.__outputBuffer = '' # the #define directive def define(self, define): - self.defines.append(define) + self.defines[define]=val # the #undef directive def undefine(self, define): - # re-map the defines list excluding the define specified in the args - self.defines[:] = [x for x in self.defines if x != define] + if define in self.defines: + self.defines.pop(define) # search: if define is defined def search_defines(self, define): - return define in self.defines: + return define in self.defines + + def evaluate(self, line): + """ + Evaluate the content of a #if, #elseif, #elif directive + + :params + line (str): definition name + + """ + return eval(line, self.defines) #returning: validness of #ifdef #else block def __if(self): @@ -143,14 +161,27 @@ def lexer(self, line): return False, True # handle #else... # handle #elseif directives + elif line[:len(self.escape) + 2] == self.escape + 'if': + if len(line.split()) > 2: + self.exit_error(self.escape + 'elseif') + else: + self.__ifblocks.append(self.evaluate(' '.join(line.split()[1:]))) + self.__ifconditions.append(' '.join(line.split()[1:])) + return False, True + + # since in version <=0.7.7, it didn't handle #if it should be #elseifdef instead. + # kept elseif with 2 elements for retro-compatibility (equivalent to #elseifdef). elif line[:len(self.escape) + 6] == self.escape + 'elseif': if len(line.split()) != 2: self.exit_error(self.escape + 'elseif') else: - self.__ifblocks[-1] = not self.__ifblocks[-1] - #self.search_defines(self.__ifconditions[-1])) - self.__ifblocks.append(self.search_defines(line.split()[1])) - self.__ifconditions.append(line.split()[1]) + if len(line.split()) == 2: + #old behaviour + self.__ifblocks.append(self.search_defines(line.split()[1])) + else: + #new behaviour + self.__ifblocks.append(self.evaluate(' '.join(line.split()[1:]))) + self.__ifconditions.append(' '.join(line.split()[1:])) return False, True # handle #else directives elif line[:len(self.escape) + 4] == self.escape + 'else': From ff0175d38c9ca8a4c8fbd9ddeb6d82318304d175 Mon Sep 17 00:00:00 2001 From: Laurent Pinson Date: Fri, 17 Dec 2021 13:26:18 +0100 Subject: [PATCH 03/13] optimized code --- pypreprocessor/__init__.py | 242 +++++++++++++++++-------------------- 1 file changed, 109 insertions(+), 133 deletions(-) diff --git a/pypreprocessor/__init__.py b/pypreprocessor/__init__.py index eded5b1..21811f5 100644 --- a/pypreprocessor/__init__.py +++ b/pypreprocessor/__init__.py @@ -14,9 +14,8 @@ class preprocessor: - def __init__(self, inFile=sys.argv[0], outFile='', defines={}, \ - removeMeta=False, escapeChar=None, mode=None, escape='#', \ - run=True, resume=False, save=True): + def __init__(self, inFile=sys.argv[0], outFile='', defines={}, removeMeta=False, + escapeChar=None, mode=None, escape='#', run=True, resume=False, save=True): # public variables #support for <=0.7.7 if isinstance(defines, collections.Sequence): @@ -76,7 +75,6 @@ def __reset_internal(self): self.__excludeblock = False self.__ifblocks = [] # contains the evaluated if conditions self.__ifconditions = [] # contains the if conditions - self.__evalsquelch = True self.__outputBuffer = '' # the #define directive @@ -103,8 +101,28 @@ def evaluate(self, line): return eval(line, self.defines) #returning: validness of #ifdef #else block - def __if(self): - return not (self.__ifblocks and all(self.__ifblocks)) + def __validate_ifs(self): + # no ifs mean we pass else check all ifs are True + return not self.__ifblocks and all(self.__ifblocks) + + def __is_directive(self, line, directive, *size): + """ + Checks the `line` is a `directive` and , if `size` is provided, checks its number of + elements is amongst the list of allowed `size` + + :params: + line (str): line to check + + directive (str): directive to be found in the `line` + + *size (int): list of allowed number of elements to compose the directive. Can be empty + + """ + if line.startswith(self.escape + directive): + if size and len(line.split()) not in size: + self.exit_error(self.escape + directive) + return True + return False # evaluate def lexer(self, line): @@ -112,131 +130,90 @@ def lexer(self, line): if not (self.__ifblocks or self.__excludeblock): if 'pypreprocessor.parse()' in line: return True, True - #this block only for faster processing (not necessary) - elif line[:len(self.escape)] != self.escape: - return False, False - # handle #define directives - if line[:len(self.escape) + 6] == self.escape + 'define': - if len(line.split()) != 2: - self.exit_error(self.escape + 'define') - else: - self.define(line.split()[1]) - return False, True - # handle #undef directives - elif line[:len(self.escape) + 5] == self.escape + 'undef': - if len(line.split()) != 2: - self.exit_error(self.escape + 'undef') - else: - self.undefine(line.split()[1]) - return False, True - # handle #exclude directives - elif line[:len(self.escape) + 7] == self.escape + 'exclude': - if len(line.split()) != 1: - self.exit_error(self.escape + 'exclude') - else: - self.__excludeblock = True - return False, True - # handle #endexclude directives - elif line[:len(self.escape) + 10] == self.escape + 'endexclude': - if len(line.split()) != 1: - self.exit_error(self.escape + 'endexclude') - else: - self.__excludeblock = False - return False, True - # handle #ifnotdef directives (is the same as: #ifdef X #else) - elif line[:len(self.escape) + 8] == self.escape + 'ifdefnot': - if len(line.split()) != 2: - self.exit_error(self.escape + 'ifdefnot') - else: - self.__ifblocks.append(not self.search_defines(line.split()[1])) - self.__ifconditions.append(line.split()[1]) - return False, True - # handle #ifdef directives - elif line[:len(self.escape) + 5] == self.escape + 'ifdef': - if len(line.split()) != 2: - self.exit_error(self.escape + 'ifdef') - else: - self.__ifblocks.append(self.search_defines(line.split()[1])) - self.__ifconditions.append(line.split()[1]) - return False, True - # handle #else... - # handle #elseif directives - elif line[:len(self.escape) + 2] == self.escape + 'if': - if len(line.split()) > 2: - self.exit_error(self.escape + 'elseif') - else: - self.__ifblocks.append(self.evaluate(' '.join(line.split()[1:]))) - self.__ifconditions.append(' '.join(line.split()[1:])) - return False, True + + # put that block here for faster processing + if not line.startswith(self.escape): #No directive --> execute + # exclude=True if we are in an exclude block or the ifs are not validated + return self.__excludeblock or not self.__validate_ifs(), False + + elif self.__is_directive(line, 'define', 2,3): + self.define(*line.split()[1:]) + + elif self.__is_directive(line, 'undef', 2): + self.undefine(line.split()[1]) + + elif self.__is_directive(line, 'exclude', 1): + self.__excludeblock = True + + elif self.__is_directive(line, 'endexclude', 1): + self.__excludeblock = False + + elif self.__is_directive(line, 'ifdefnot', 2): + self.__ifblocks.append(not self.search_defines(line.split()[1])) + self.__ifconditions.append(line.split()[1]) + + elif self.__is_directive(line, 'ifdef', 2): + self.__ifblocks.append(self.search_defines(line.split()[1])) + self.__ifconditions.append(line.split()[1]) + + elif self.__is_directive(line, 'if'): + self.__ifblocks.append(self.evaluate(' '.join(line.split()[1:]))) + self.__ifconditions.append(' '.join(line.split()[1:])) # since in version <=0.7.7, it didn't handle #if it should be #elseifdef instead. # kept elseif with 2 elements for retro-compatibility (equivalent to #elseifdef). - elif line[:len(self.escape) + 6] == self.escape + 'elseif': - if len(line.split()) != 2: - self.exit_error(self.escape + 'elseif') - else: - if len(line.split()) == 2: - #old behaviour - self.__ifblocks.append(self.search_defines(line.split()[1])) - else: - #new behaviour - self.__ifblocks.append(self.evaluate(' '.join(line.split()[1:]))) - self.__ifconditions.append(' '.join(line.split()[1:])) - return False, True - # handle #else directives - elif line[:len(self.escape) + 4] == self.escape + 'else': - if len(line.split()) != 1: - self.exit_error(self.escape + 'else') - else: - self.__ifblocks[-1] = not self.__ifblocks[-1] - #self.search_defines(self.__ifconditions[-1])) - return False, True - # handle #endif.. - # handle #endififdef - elif line[:len(self.escape) + 10] == self.escape + 'endififdef': - if len(line.split()) != 2: - self.exit_error(self.escape + 'endififdef') + elif self.__is_directive(line, 'elseif'): + # do else + self.__ifblocks[-1] = not self.__ifblocks[-1] + # do if + if len(line.split()) == 2: + #old behaviour + self.__ifblocks.append(self.search_defines(line.split()[1])) else: - if len(self.__ifconditions) >= 1: + #new behaviour + self.__ifblocks.append(self.evaluate(' '.join(line.split()[1:]))) + self.__ifconditions.append(' '.join(line.split()[1:])) + + elif self.__is_directive(line, 'else', 1): + self.__ifblocks[-1] = not self.__ifblocks[-1] #opposite of last if + + elif self.__is_directive(line, 'endififdef', 2): + # do endif + if len(self.__ifconditions) >= 1: + self.__ifblocks.pop(-1) + self.__ifconditions.pop(-1) + # do ifdef + self.__ifblocks.append(self.search_defines(line.split()[1])) + self.__ifconditions.append(line.split()[1]) + + elif self.__is_directive(line, 'endifall', 1): + self.__ifblocks = [] + self.__ifconditions = [] + + # handle #endif and #endif directives + elif self.__is_directive(line, 'endif', 1): + try: + number = int(line[6:]) + except ValueError as VE: + number = 1 + + if len(self.__ifconditions) >= number: + for i in range(0, number): self.__ifblocks.pop(-1) - self.__ifcondition = self.__ifconditions.pop(-1) - else: - self.__ifblocks = [] - self.__ifconditions = [] - self.__ifblocks.append(self.search_defines(line.split()[1])) - self.__ifconditions.append(line.split()[1]) - return False, True - # handle #endifall directives - elif line[:len(self.escape) + 8] == self.escape + 'endifall': - if len(line.split()) != 1: - self.exit_error(self.escape + 'endifall') + self.__ifconditions.pop(-1) else: + print('Warning trying to remove more blocks than present', \ + self.input, self.__linenum) self.__ifblocks = [] self.__ifconditions = [] - return False, True - # handle #endif and #endif numb directives - elif line[:len(self.escape) + 5] == self.escape + 'endif': - if len(line.split()) != 1: - self.exit_error(self.escape + 'endif number') - else: - try: - number = int(line[6:]) - except ValueError as VE: - #print('ValueError',VE) - #self.exit_error(self.escape + 'endif number') - number = 1 - if len(self.__ifconditions) > number: - for i in range(0, number): - self.__ifblocks.pop(-1) - self.__ifcondition = self.__ifconditions.pop(-1) - else: - print('Warning try to remove more blocks than present', \ - self.input, self.__linenum) - self.__ifblocks = [] - self.__ifconditions = [] - return False, True - else: #No directive --> execute - return self.__excludeblock or self.__ifblocks, False + + else: + # unknown directive or comment + if len(line.split()[0]) > 1: + print('Warning unknown directive or comment starting with ', \ + line.split()[0], self.input, self.__linenum) + + return False, True # error handling def exit_error(self, directive): @@ -258,22 +235,19 @@ def parse(self): # open the input file try: with io.open(os.path.join(self.input), 'r', encoding=self.readEncoding) as input_file: - # process the input file - for line in input_file: - self.__linenum += 1 - # to squelch or not to squelch - squelch, metaData = self.lexer(line) + for self.__linenum, line in enumerate(input_file): + exclude, metaData = self.lexer(line) # process and output - if self.removeMeta is True: - if metaData is True or squelch is True: + if self.removeMeta: + if metaData or exclude: continue - if squelch is True: + if exclude: if metaData: self.__outputBuffer += self.escape + line else: self.__outputBuffer += self.escape[0] + line continue - if squelch is False: + else: self.__outputBuffer += line continue finally: @@ -286,7 +260,7 @@ def parse(self): except SyntaxError: select = 'no' select = select.lower() - if select in ('yes', 'true', 'y', '1'): + if select.lower() in ('yes', 'true', 'y', '1'): print('Name of input and output file: ', self.input, ' ', self.output) for i, item in enumerate(self.__ifconditions): if (item in self.defines) != self.__ifblocks[i]: @@ -315,10 +289,12 @@ def post_process(self): self.override_import() else: self.on_the_fly() + if not self.save: # remove tmp file if os.path.exists(self.output): os.remove(self.output) + if not self.resume: # break execution so python doesn't # run the rest of the pre-processed code From fad8cc34788dbb32fcacd3083e8c3d120d267a11 Mon Sep 17 00:00:00 2001 From: Laurent Pinson Date: Fri, 17 Dec 2021 13:30:30 +0100 Subject: [PATCH 04/13] documentation + typos --- pypreprocessor/__init__.py | 103 +++++++++++++++++++++++++++++++------ 1 file changed, 88 insertions(+), 15 deletions(-) diff --git a/pypreprocessor/__init__.py b/pypreprocessor/__init__.py index 21811f5..166831e 100644 --- a/pypreprocessor/__init__.py +++ b/pypreprocessor/__init__.py @@ -3,7 +3,7 @@ __author__ = 'Evan Plaice' __coauthor__ = 'Hendi O L, Epikem' -__version__ = '0.7.7' +__version__ = '0.8' import sys import os @@ -37,6 +37,10 @@ def __init__(self, inFile=sys.argv[0], outFile='', defines={}, removeMeta=False, self.__reset_internal() def check_deprecation(self): + """ + Deprecation checks for older implementation of this library + + """ def deprecation(message): import warnings warnings.simplefilter('always', DeprecationWarning) @@ -69,7 +73,6 @@ def deprecation(message): print('Unknown mode : ' + str(self.mode)) deprecation(msg) - # reseting internal things to parse a second file def __reset_internal(self): self.__linenum = 0 self.__excludeblock = False @@ -77,17 +80,38 @@ def __reset_internal(self): self.__ifconditions = [] # contains the if conditions self.__outputBuffer = '' - # the #define directive - def define(self, define): + def define(self, define, val=None): + """ + Adds variable definition to the store as expected from a #defined directive. + The directive can contains no value as it would be tested with a #ifdef directive or + with a value for an evaluation as in an #if directive. + + :params + define (str): definition name + + val (str): definition value when it exists. Default is None + """ self.defines[define]=val - # the #undef directive def undefine(self, define): + """ + Removes variable definition from store as expected from an #undef directive + + :params + define (str): definition name + + """ if define in self.defines: self.defines.pop(define) - # search: if define is defined def search_defines(self, define): + """ + Checks variable is defined as used in #ifdef, #ifnotdef & #elseif directives + + :params + define (str): definition name + + """ return define in self.defines def evaluate(self, line): @@ -100,10 +124,16 @@ def evaluate(self, line): """ return eval(line, self.defines) - #returning: validness of #ifdef #else block def __validate_ifs(self): + """ + Evaluate if the successive #ifs block are validated for the current position + + :return + ifs (bool): True if all ifs condition are validated + + """ # no ifs mean we pass else check all ifs are True - return not self.__ifblocks and all(self.__ifblocks) + return not self.__ifblocks or all(self.__ifblocks) def __is_directive(self, line, directive, *size): """ @@ -124,9 +154,20 @@ def __is_directive(self, line, directive, *size): return True return False - # evaluate def lexer(self, line): - # return values are (squelch, metadata) + """ + Analyse the `line`. This method attempts to find a known directive and, when found, to + understand it and to perform appropriate action. + + :params + line (str): line of code to analyse + + :return + exclude (bool): should the line be excluded in the final output? + + metadata (bool): is this line a directive? + + """ if not (self.__ifblocks or self.__excludeblock): if 'pypreprocessor.parse()' in line: return True, True @@ -217,19 +258,37 @@ def lexer(self, line): # error handling def exit_error(self, directive): + """ + Prints error and interrupts execution + + :params + directive (str): faulty directive + + """ print('File: "' + self.input + '", line ' + str(self.__linenum)) print('SyntaxError: Invalid ' + directive + ' directive') sys.exit(1) def rewrite_traceback(self): + """ + Dumps traceback but with the input file name included + + """ trace = traceback.format_exc().splitlines() trace[-2]=trace[-2].replace("", self.input)) for line in trace: print(line) - - # parsing/processing def parse(self): + """ + Main method: + - reset internal counters/values + - check & warn about deprecation + - starts the parsing of the input file + - warn of unclosed #ifdef blocks if any + - trigger post-process activities + + """ self.__reset_internal() self.check_deprecation() # open the input file @@ -259,7 +318,6 @@ def parse(self): select = input('Do you want more Information? ') except SyntaxError: select = 'no' - select = select.lower() if select.lower() in ('yes', 'true', 'y', '1'): print('Name of input and output file: ', self.input, ' ', self.output) for i, item in enumerate(self.__ifconditions): @@ -270,11 +328,18 @@ def parse(self): print('Block:', item, ' is in condition: ', cond) self.post_process() - # post-processor def post_process(self): + """ + Perform post-parsing activities: + - write output file from parsing. + - override import if requested or attempt execution with its content + - remove output file if no save was requested + - force exit if resume was not requested + + """ try: # set file name - if self.output == '': + if not self.output: self.output = self.input[0:-len(self.input.split('.')[-1])-1]+'_out.'+self.input.split('.')[-1] # write post-processed code to file @@ -302,6 +367,10 @@ def post_process(self): # postprocessor - override an import def override_import(self): + """ + Override the import of the output of the processed file + + """ try: moduleName = self.input.split('.')[0] tmpModuleName = self.output.split('.')[0] @@ -317,6 +386,10 @@ def override_import(self): # postprocessor - on-the-fly execution def on_the_fly(self): + """ + Execute output of the processed file + + """ try: with io.open(self.output, "r", encoding=self.readEncoding) as f: exec(f.read()) From 5f42487ce243366f2a3995a077c93ed761d5097c Mon Sep 17 00:00:00 2001 From: Laurent Pinson Date: Fri, 17 Dec 2021 13:32:33 +0100 Subject: [PATCH 05/13] add #elseifdef, #elif, #ifnotdef, #ifndef --- pypreprocessor/__init__.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/pypreprocessor/__init__.py b/pypreprocessor/__init__.py index 166831e..9f359b2 100644 --- a/pypreprocessor/__init__.py +++ b/pypreprocessor/__init__.py @@ -189,7 +189,10 @@ def lexer(self, line): elif self.__is_directive(line, 'endexclude', 1): self.__excludeblock = False - elif self.__is_directive(line, 'ifdefnot', 2): + # #ifnotdef sounds better than #ifdefnot.. + elif self.__is_directive(line, 'ifdefnot', 2) or \ + self.__is_directive(line, 'ifnotdef', 2) or \ + self.__is_directive(line, 'ifndef', 2): self.__ifblocks.append(not self.search_defines(line.split()[1])) self.__ifconditions.append(line.split()[1]) @@ -203,7 +206,8 @@ def lexer(self, line): # since in version <=0.7.7, it didn't handle #if it should be #elseifdef instead. # kept elseif with 2 elements for retro-compatibility (equivalent to #elseifdef). - elif self.__is_directive(line, 'elseif'): + elif self.__is_directive(line, 'elseif') or \ + self.__is_directive(line, 'elif'): # do else self.__ifblocks[-1] = not self.__ifblocks[-1] # do if @@ -214,7 +218,14 @@ def lexer(self, line): #new behaviour self.__ifblocks.append(self.evaluate(' '.join(line.split()[1:]))) self.__ifconditions.append(' '.join(line.split()[1:])) - + + elif self.__is_directive(line, 'elseifdef', 2): + # do else + self.__ifblocks[-1] = not self.__ifblocks[-1] + # do ifdef + self.__ifblocks.append(self.search_defines(line.split()[1])) + self.__ifconditions.append(line.split()[1]) + elif self.__is_directive(line, 'else', 1): self.__ifblocks[-1] = not self.__ifblocks[-1] #opposite of last if From 5fb26d471be00d925286294824cfbf1c0d177209 Mon Sep 17 00:00:00 2001 From: Laurent Pinson Date: Mon, 20 Dec 2021 10:58:37 +0100 Subject: [PATCH 06/13] support for version <=0.7.7 --- pypreprocessor/__init__.py | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/pypreprocessor/__init__.py b/pypreprocessor/__init__.py index 9f359b2..9e17d4f 100644 --- a/pypreprocessor/__init__.py +++ b/pypreprocessor/__init__.py @@ -12,15 +12,24 @@ import io import collections +#support for <=0.7.7 +class customDict(dict): + def append(self, var): + self[var]=None + class preprocessor: def __init__(self, inFile=sys.argv[0], outFile='', defines={}, removeMeta=False, escapeChar=None, mode=None, escape='#', run=True, resume=False, save=True): # public variables + self.defines = customDict() #support for <=0.7.7 if isinstance(defines, collections.Sequence): - defines = {x:None for x in defines} - self.defines = defines + for x in defines: + self.defines.append(x) + else: + for x,y in defines: + self.define(x,y) self.input = inFile self.output = outFile self.removeMeta = removeMeta @@ -30,7 +39,7 @@ def __init__(self, inFile=sys.argv[0], outFile='', defines={}, removeMeta=False, self.run = run self.resume = resume self.save = save - self.readEncoding = sys.stdin.encoding + self.readEncoding = sys.stdin.encoding self.writeEncoding = sys.stdout.encoding # private variables @@ -91,6 +100,12 @@ def define(self, define, val=None): val (str): definition value when it exists. Default is None """ + # try conversion for number else evaluate() might fail + try: + val = int(val) + except: + # assume val is string + pass self.defines[define]=val def undefine(self, define): @@ -122,7 +137,11 @@ def evaluate(self, line): line (str): definition name """ - return eval(line, self.defines) + try: + return eval(line, self.defines) + except BaseException as e: + print(str(e)) + self.exit_error(self.escape + 'if') def __validate_ifs(self): """ @@ -261,9 +280,12 @@ def lexer(self, line): else: # unknown directive or comment - if len(line.split()[0]) > 1: + # escapechar + space ==> comment + # starts with #!/ ==> shebang + # else print warning + if len(line.split()[0]) > 1 and not line.startswith('#!/'): print('Warning unknown directive or comment starting with ', \ - line.split()[0], self.input, self.__linenum) + line.split()[0], self.input, self.__linenum + 1) return False, True @@ -276,7 +298,7 @@ def exit_error(self, directive): directive (str): faulty directive """ - print('File: "' + self.input + '", line ' + str(self.__linenum)) + print('File: "' + self.input + '", line ' + str(self.__linenum + 1)) print('SyntaxError: Invalid ' + directive + ' directive') sys.exit(1) @@ -286,7 +308,7 @@ def rewrite_traceback(self): """ trace = traceback.format_exc().splitlines() - trace[-2]=trace[-2].replace("", self.input)) + trace[-2]=trace[-2].replace("", self.input) for line in trace: print(line) From 03ce4fff06cea8d9419520f88a08216192638f5f Mon Sep 17 00:00:00 2001 From: Laurent Pinson Date: Mon, 20 Dec 2021 10:58:59 +0100 Subject: [PATCH 07/13] add __main__ for execution as python -m pypreprocessor ... --- pypreprocessor/__main__.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 pypreprocessor/__main__.py diff --git a/pypreprocessor/__main__.py b/pypreprocessor/__main__.py new file mode 100644 index 0000000..ce29f93 --- /dev/null +++ b/pypreprocessor/__main__.py @@ -0,0 +1,33 @@ +import argparse +from . import preprocessor + +parser = argparse.ArgumentParser( + description="Check https://github.com/interpreters/pypreprocessor for documentation", + epilog="""Examples: + python -m """+__package__+""" somepython.py post-proc.py -m -d DEBUG FILE:4 EXEC + python -m """+__package__+""" table.json --escape #@ -d NUM:4 ID:1 + """, + formatter_class=argparse.RawDescriptionHelpFormatter, + usage=' python -m '+__package__+' input [output] [-h] [-r] [-m] [-e ESCAPE] [-d [DEFINE ...]]', +) +parser.add_argument("-r", "--run", help="Run on the fly", + action='store_true', default=False) +parser.add_argument("-m", "--removeMeta", help="remove meta lines from the output", + action='store_true', default=False) +parser.add_argument("-e", "--escape", help="define the escape sequence to use. Default is #") +parser.add_argument("-d", "--define", help="list of variable to define", nargs='*') +parser.add_argument("input", help="input file.") +parser.add_argument("output", nargs='?', help="output file. Default is _out.") +args = parser.parse_args() + +p=preprocessor(inFile=args.input, mode=None, removeMeta=args.removeMeta, escapeChar=None, + run=args.run, resume=False, save=True) +if args.output: + p.define = args.output +if args.define: + for elem in args.define: + p.define(*elem.split(':')) +if args.escape: + p.escape = args.escape + +p.parse() \ No newline at end of file From b201217e7777de6e78b1085dfe5a9ec4dfad6bec Mon Sep 17 00:00:00 2001 From: Laurent Pinson Date: Mon, 20 Dec 2021 14:02:00 +0100 Subject: [PATCH 08/13] package infos --- MANIFEST.in | 1 + README.md | 5 +++++ pypreprocessor/__init__.py | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 6177c41..d18b373 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,3 +4,4 @@ include INSTALL.md include setup.py include tests/__init__.py include pypreprocessor/__init__.py +include pypreprocessor/__main__.py diff --git a/README.md b/README.md index 281ca09..b689dd1 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,11 @@ The syntax for pypreprocessor uses a select subset of the stanard c-style prepro #ifdef constant ``` +* makes the subsequent block of code available if the specified condition is set +```python +#if boolean_condition +``` + * makes the subsequent block of code available if all of the preceding #ifdef statements return false ```python #else diff --git a/pypreprocessor/__init__.py b/pypreprocessor/__init__.py index 9e17d4f..7b95ae0 100644 --- a/pypreprocessor/__init__.py +++ b/pypreprocessor/__init__.py @@ -2,7 +2,7 @@ # pypreprocessor.py __author__ = 'Evan Plaice' -__coauthor__ = 'Hendi O L, Epikem' +__coauthor__ = 'Hendi O L, Epikem, Laurent Pinson' __version__ = '0.8' import sys From a74becf7d720e6bc77350cbf7f36dd3da1f25c19 Mon Sep 17 00:00:00 2001 From: Laurent Pinson Date: Tue, 21 Dec 2021 10:58:48 +0100 Subject: [PATCH 09/13] added overload option, improved evaluation of lines --- pypreprocessor/__init__.py | 70 +++++++++++++++++++++++++++----------- pypreprocessor/__main__.py | 4 ++- 2 files changed, 53 insertions(+), 21 deletions(-) diff --git a/pypreprocessor/__init__.py b/pypreprocessor/__init__.py index 7b95ae0..d12c7c7 100644 --- a/pypreprocessor/__init__.py +++ b/pypreprocessor/__init__.py @@ -11,22 +11,24 @@ import imp import io import collections +import re #support for <=0.7.7 class customDict(dict): def append(self, var): - self[var]=None + self[var]=True class preprocessor: def __init__(self, inFile=sys.argv[0], outFile='', defines={}, removeMeta=False, - escapeChar=None, mode=None, escape='#', run=True, resume=False, save=True): + escapeChar=None, mode=None, escape='#', run=True, resume=False, + save=True, overload=True): # public variables self.defines = customDict() #support for <=0.7.7 if isinstance(defines, collections.Sequence): for x in defines: - self.defines.append(x) + self.define(x) else: for x,y in defines: self.define(x,y) @@ -39,6 +41,7 @@ def __init__(self, inFile=sys.argv[0], outFile='', defines={}, removeMeta=False, self.run = run self.resume = resume self.save = save + self.overload = overload self.readEncoding = sys.stdin.encoding self.writeEncoding = sys.stdout.encoding @@ -88,15 +91,20 @@ def __reset_internal(self): self.__ifblocks = [] # contains the evaluated if conditions self.__ifconditions = [] # contains the if conditions self.__outputBuffer = '' + self.__overloaded = list(self.defines.keys()) if self.overload else [] - def define(self, define, val=None): + + def define(self, name, val=True): """ - Adds variable definition to the store as expected from a #defined directive. + Adds variable definition to the store as expected from a #define directive. The directive can contains no value as it would be tested with a #ifdef directive or with a value for an evaluation as in an #if directive. + + Note: if the `name` was part of the initial definition and `overload` was set to + True, this new definition will be skipped :params - define (str): definition name + name (str): definition name val (str): definition value when it exists. Default is None """ @@ -106,7 +114,8 @@ def define(self, define, val=None): except: # assume val is string pass - self.defines[define]=val + if name not in self.__overloaded: + self.defines[name]=val def undefine(self, define): """ @@ -119,7 +128,7 @@ def undefine(self, define): if define in self.defines: self.defines.pop(define) - def search_defines(self, define): + def __is_defined(self, define): """ Checks variable is defined as used in #ifdef, #ifnotdef & #elseif directives @@ -129,7 +138,7 @@ def search_defines(self, define): """ return define in self.defines - def evaluate(self, line): + def __evaluate_if(self, line): """ Evaluate the content of a #if, #elseif, #elif directive @@ -138,7 +147,9 @@ def evaluate(self, line): """ try: - return eval(line, self.defines) + # replace C-style bool format by Python's + line = line.replace('&&', 'and').replace('||', 'or').replace('!','not ') + return eval(line, self.defines) or False except BaseException as e: print(str(e)) self.exit_error(self.escape + 'if') @@ -173,6 +184,21 @@ def __is_directive(self, line, directive, *size): return True return False + def __cleanup_line(self, line): + """ + Clean a line of anything that should not impact parsing such as C-style comment + + :params: + line (str): line to check + + :return + line (str): cleaned line + + """ + line= re.sub('\s*/\*.*\*/\s+', '', line) #remove /* */ C-style comment + line= re.sub('\s*//.*', '', line) #remove // C-style comment + return line + def lexer(self, line): """ Analyse the `line`. This method attempts to find a known directive and, when found, to @@ -187,6 +213,7 @@ def lexer(self, line): metadata (bool): is this line a directive? """ + line = line.strip() if not (self.__ifblocks or self.__excludeblock): if 'pypreprocessor.parse()' in line: return True, True @@ -196,7 +223,10 @@ def lexer(self, line): # exclude=True if we are in an exclude block or the ifs are not validated return self.__excludeblock or not self.__validate_ifs(), False - elif self.__is_directive(line, 'define', 2,3): + # strip line of any C-style comment + line = self.__cleanup_line(line) + + if self.__is_directive(line, 'define', 2,3): self.define(*line.split()[1:]) elif self.__is_directive(line, 'undef', 2): @@ -212,15 +242,15 @@ def lexer(self, line): elif self.__is_directive(line, 'ifdefnot', 2) or \ self.__is_directive(line, 'ifnotdef', 2) or \ self.__is_directive(line, 'ifndef', 2): - self.__ifblocks.append(not self.search_defines(line.split()[1])) + self.__ifblocks.append(not self.__is_defined(line.split()[1])) self.__ifconditions.append(line.split()[1]) elif self.__is_directive(line, 'ifdef', 2): - self.__ifblocks.append(self.search_defines(line.split()[1])) + self.__ifblocks.append(self.__is_defined(line.split()[1])) self.__ifconditions.append(line.split()[1]) elif self.__is_directive(line, 'if'): - self.__ifblocks.append(self.evaluate(' '.join(line.split()[1:]))) + self.__ifblocks.append(self.__evaluate_if(' '.join(line.split()[1:]))) self.__ifconditions.append(' '.join(line.split()[1:])) # since in version <=0.7.7, it didn't handle #if it should be #elseifdef instead. @@ -232,18 +262,18 @@ def lexer(self, line): # do if if len(line.split()) == 2: #old behaviour - self.__ifblocks.append(self.search_defines(line.split()[1])) + self.__ifblocks[-1] = self.__is_defined(line.split()[1]) else: #new behaviour - self.__ifblocks.append(self.evaluate(' '.join(line.split()[1:]))) + self.__ifblocks[-1] = self.__evaluate_if(' '.join(line.split()[1:])) self.__ifconditions.append(' '.join(line.split()[1:])) elif self.__is_directive(line, 'elseifdef', 2): # do else self.__ifblocks[-1] = not self.__ifblocks[-1] - # do ifdef - self.__ifblocks.append(self.search_defines(line.split()[1])) - self.__ifconditions.append(line.split()[1]) + # do if + self.__ifblocks[-1]=self.__is_defined(line.split()[1]) + self.__ifconditions.append(' '.join(line.split()[1:])) elif self.__is_directive(line, 'else', 1): self.__ifblocks[-1] = not self.__ifblocks[-1] #opposite of last if @@ -254,7 +284,7 @@ def lexer(self, line): self.__ifblocks.pop(-1) self.__ifconditions.pop(-1) # do ifdef - self.__ifblocks.append(self.search_defines(line.split()[1])) + self.__ifblocks.append(self.__is_defined(line.split()[1])) self.__ifconditions.append(line.split()[1]) elif self.__is_directive(line, 'endifall', 1): diff --git a/pypreprocessor/__main__.py b/pypreprocessor/__main__.py index ce29f93..4f74abd 100644 --- a/pypreprocessor/__main__.py +++ b/pypreprocessor/__main__.py @@ -16,12 +16,14 @@ action='store_true', default=False) parser.add_argument("-e", "--escape", help="define the escape sequence to use. Default is #") parser.add_argument("-d", "--define", help="list of variable to define", nargs='*') +parser.add_argument("-o", "--overload", help="overload variable definition in the file by those \ + provided by --define", action='store_true', default=False) parser.add_argument("input", help="input file.") parser.add_argument("output", nargs='?', help="output file. Default is _out.") args = parser.parse_args() p=preprocessor(inFile=args.input, mode=None, removeMeta=args.removeMeta, escapeChar=None, - run=args.run, resume=False, save=True) + run=args.run, resume=False, save=True, overload=args.overload) if args.output: p.define = args.output if args.define: From f49ffa1d57f96482dda1794d71accd53ee8a77de Mon Sep 17 00:00:00 2001 From: Laurent Pinson Date: Tue, 21 Dec 2021 10:58:58 +0100 Subject: [PATCH 10/13] README updated --- README.md | 81 +++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 73 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index b689dd1..eac2d2b 100644 --- a/README.md +++ b/README.md @@ -55,17 +55,16 @@ ## Syntax -The syntax for pypreprocessor uses a select subset of the stanard c-style preprocessor directives, and then some... +The syntax for pypreprocessor uses a select subset of the standard c-style preprocessor directives, and then some... **Supported directives** -* define non-value constants used by the preprocessor +* defines constants (valued or not) used by the preprocessor ```python -#define constant +#define constant [value] ``` - -* remove a non-value constant from the list of defined constants +* removes a constant from the list of defined constants ```python #undef constant ``` @@ -75,26 +74,46 @@ The syntax for pypreprocessor uses a select subset of the stanard c-style prepro #ifdef constant ``` -* makes the subsequent block of code available if the specified condition is set +* makes the subsequent block of code available if the specified constant is not set +```python +#ifndef constant +``` + +* makes the subsequent block of code available if the specified condition returns true ```python #if boolean_condition ``` -* makes the subsequent block of code available if all of the preceding #ifdef statements return false +* makes the subsequent block of code available if all of the preceding #ifdef, #elif, #if statements returns false ```python #else ``` +* makes the subsequent block of code available if all of the preceding #ifdef, #elif, #if statements return false and the specifified condition returns true +```python +#elif boolean_condition +``` + * required to close out an #ifdef/#else block ```python #endif ``` + +* Interrupts execution and returns error when reached +```python +#error +``` + +**Unofficial supported directives** + +Unofficial directives exist to ease writing long files but should not be used in file that could be preprocessed without pypreprocessor + * possibility to close all open blocks ```python #endifall ``` -* exclude the subsequent block of code (conditionals not included). I know it doesn't fit into the standard set of c-style directives but it's too handy to exclude (no pun). +* excludes the subsequent block of code (conditionals not included). I know it doesn't fit into the standard set of c-style directives but it's too handy to exclude (no pun). ```python #exclude ``` @@ -104,6 +123,50 @@ The syntax for pypreprocessor uses a select subset of the stanard c-style prepro #endexclude ``` +* Attempts closing #ifdef/#else blocks +```python +#endif +``` + +* Similar to #ifndef +```python +#ifnotdef constant +``` + +* Similar to #ifndef +```python +#ifdefnot constant +``` + +* Similar to #elif boolean_condition +```python +#elseif boolean_condition +``` + +* Similar to #elif constant +```python +#elseifdef constant +``` + +* Similar to #endif followed by #ifdef constant +```python +#endififdef constant +``` + +**Unsupported directives** + +Unsupported directives are not handled by pypreprocessor and concidered as comment + +* Inserts a particuliar header from another file. This has no use in Python +```python +#include +``` + +* Issues special commands to the compiler, using a standardized method. This has no use in Python +```python +#pragma +``` + **Options** The following options need to be set prior to pypreprocessor.parse() @@ -118,12 +181,14 @@ add defines to the preprocessor programmatically, this allows the source file to pypreprocessor.run = True / False pypreprocessor.resume = True / False pypreprocessor.save = True / False +pypreprocessor.overload = True / False ``` set the options of the preprocessor: * run: Run the preprocessed code if true. Default is true * resume: Return after a file is preprocessed and can preprocess a next file if true. Default is false * save: Save preprocessed code if true. Default is true +* overload: Any defines added to the preprocessor will overload existing defines. Default is false ```python pypreprocessor.input = 'inputFile.py' From 40d01920b3ca2eac6d9e68f93c6d2b8608aeeb41 Mon Sep 17 00:00:00 2001 From: Laurent Pinson Date: Tue, 21 Dec 2021 11:33:42 +0100 Subject: [PATCH 11/13] Added quiet mode --- README.md | 2 ++ pypreprocessor/__init__.py | 14 ++++++++------ pypreprocessor/__main__.py | 5 ++++- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index eac2d2b..7c812d5 100644 --- a/README.md +++ b/README.md @@ -182,6 +182,7 @@ pypreprocessor.run = True / False pypreprocessor.resume = True / False pypreprocessor.save = True / False pypreprocessor.overload = True / False +pypreprocessor.quiet = True / False ``` set the options of the preprocessor: @@ -189,6 +190,7 @@ set the options of the preprocessor: * resume: Return after a file is preprocessed and can preprocess a next file if true. Default is false * save: Save preprocessed code if true. Default is true * overload: Any defines added to the preprocessor will overload existing defines. Default is false +* quiet: no warning about ununderstood directives or missing #indef ```python pypreprocessor.input = 'inputFile.py' diff --git a/pypreprocessor/__init__.py b/pypreprocessor/__init__.py index d12c7c7..3373ee9 100644 --- a/pypreprocessor/__init__.py +++ b/pypreprocessor/__init__.py @@ -22,7 +22,7 @@ def append(self, var): class preprocessor: def __init__(self, inFile=sys.argv[0], outFile='', defines={}, removeMeta=False, escapeChar=None, mode=None, escape='#', run=True, resume=False, - save=True, overload=True): + save=True, overload=True, quiet=False): # public variables self.defines = customDict() #support for <=0.7.7 @@ -42,6 +42,7 @@ def __init__(self, inFile=sys.argv[0], outFile='', defines={}, removeMeta=False, self.resume = resume self.save = save self.overload = overload + self.quiet = quiet self.readEncoding = sys.stdin.encoding self.writeEncoding = sys.stdout.encoding @@ -303,8 +304,9 @@ def lexer(self, line): self.__ifblocks.pop(-1) self.__ifconditions.pop(-1) else: - print('Warning trying to remove more blocks than present', \ - self.input, self.__linenum) + if not self.quiet: + print('Warning trying to remove more blocks than present', + self.input, self.__linenum) self.__ifblocks = [] self.__ifconditions = [] @@ -313,8 +315,8 @@ def lexer(self, line): # escapechar + space ==> comment # starts with #!/ ==> shebang # else print warning - if len(line.split()[0]) > 1 and not line.startswith('#!/'): - print('Warning unknown directive or comment starting with ', \ + if len(line.split()[0]) > 1 and not line.startswith('#!/') and not self.quiet: + print('Warning unknown directive or comment starting with ', line.split()[0], self.input, self.__linenum + 1) return False, True @@ -374,7 +376,7 @@ def parse(self): continue finally: #Warnings for unclosed #ifdef blocks - if self.__ifblocks: + if self.__ifblocks and not self.quiet: print('Warning: Number of unclosed Ifdefblocks: ', len(self.__ifblocks)) print('Can cause unwished behaviour in the preprocessed code, preprocessor is safe') try: diff --git a/pypreprocessor/__main__.py b/pypreprocessor/__main__.py index 4f74abd..437f87b 100644 --- a/pypreprocessor/__main__.py +++ b/pypreprocessor/__main__.py @@ -18,17 +18,20 @@ parser.add_argument("-d", "--define", help="list of variable to define", nargs='*') parser.add_argument("-o", "--overload", help="overload variable definition in the file by those \ provided by --define", action='store_true', default=False) +parser.add_argument("-q", "--quiet", help="No warning on ununderstable directives and missign ending", + action='store_true', default=False) parser.add_argument("input", help="input file.") parser.add_argument("output", nargs='?', help="output file. Default is _out.") args = parser.parse_args() p=preprocessor(inFile=args.input, mode=None, removeMeta=args.removeMeta, escapeChar=None, - run=args.run, resume=False, save=True, overload=args.overload) + run=args.run, resume=False, save=True, overload=args.overload, quiet=args.quiet) if args.output: p.define = args.output if args.define: for elem in args.define: p.define(*elem.split(':')) + self.__reset_internal() # not very pythonic if args.escape: p.escape = args.escape From 2d059916d9412bc758dd00e8148c35559213cc02 Mon Sep 17 00:00:00 2001 From: Laurent Pinson Date: Tue, 21 Dec 2021 16:13:52 +0100 Subject: [PATCH 12/13] solved issue #31 --- README.md | 2 +- pypreprocessor/__init__.py | 66 +++++++++++++++++++++----------------- pypreprocessor/__main__.py | 14 +++----- 3 files changed, 42 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 7c812d5..53aedbf 100644 --- a/README.md +++ b/README.md @@ -190,7 +190,7 @@ set the options of the preprocessor: * resume: Return after a file is preprocessed and can preprocess a next file if true. Default is false * save: Save preprocessed code if true. Default is true * overload: Any defines added to the preprocessor will overload existing defines. Default is false -* quiet: no warning about ununderstood directives or missing #indef +* quiet: no warning about not understood directives or missing #indef ```python pypreprocessor.input = 'inputFile.py' diff --git a/pypreprocessor/__init__.py b/pypreprocessor/__init__.py index 3373ee9..04542fe 100644 --- a/pypreprocessor/__init__.py +++ b/pypreprocessor/__init__.py @@ -3,7 +3,7 @@ __author__ = 'Evan Plaice' __coauthor__ = 'Hendi O L, Epikem, Laurent Pinson' -__version__ = '0.8' +__version__ = '1.0' import sys import os @@ -20,17 +20,19 @@ def append(self, var): class preprocessor: + __overloaded = [] + defines = customDict() + def __init__(self, inFile=sys.argv[0], outFile='', defines={}, removeMeta=False, escapeChar=None, mode=None, escape='#', run=True, resume=False, save=True, overload=True, quiet=False): # public variables - self.defines = customDict() - #support for <=0.7.7 + # support for <=0.7.7 if isinstance(defines, collections.Sequence): for x in defines: - self.define(x) + self.define(*x.split(':')) else: - for x,y in defines: + for x,y in defines.items(): self.define(x,y) self.input = inFile self.output = outFile @@ -89,12 +91,16 @@ def deprecation(message): def __reset_internal(self): self.__linenum = 0 self.__excludeblock = False - self.__ifblocks = [] # contains the evaluated if conditions - self.__ifconditions = [] # contains the if conditions + # contains the evaluated if conditions + # due to the introduction of #elif, elements of __ifblocks are duos of boolean + # the 1st is the evaluation of the current #if or #elif or #else + # the 2nd indicates if at least one #if or #elif was True in the whole #if/#endif block + self.__ifblocks = [] + # contains the if conditions + self.__ifconditions = [] self.__outputBuffer = '' self.__overloaded = list(self.defines.keys()) if self.overload else [] - def define(self, name, val=True): """ Adds variable definition to the store as expected from a #define directive. @@ -164,7 +170,7 @@ def __validate_ifs(self): """ # no ifs mean we pass else check all ifs are True - return not self.__ifblocks or all(self.__ifblocks) + return not self.__ifblocks or all(x[0] for x in self.__ifblocks) def __is_directive(self, line, directive, *size): """ @@ -243,41 +249,43 @@ def lexer(self, line): elif self.__is_directive(line, 'ifdefnot', 2) or \ self.__is_directive(line, 'ifnotdef', 2) or \ self.__is_directive(line, 'ifndef', 2): - self.__ifblocks.append(not self.__is_defined(line.split()[1])) + _check = not self.__is_defined(line.split()[1]) + self.__ifblocks.append([ _check, _check]) self.__ifconditions.append(line.split()[1]) elif self.__is_directive(line, 'ifdef', 2): - self.__ifblocks.append(self.__is_defined(line.split()[1])) + _check = self.__is_defined(line.split()[1]) + self.__ifblocks.append([ _check, _check]) self.__ifconditions.append(line.split()[1]) elif self.__is_directive(line, 'if'): - self.__ifblocks.append(self.__evaluate_if(' '.join(line.split()[1:]))) + _check = self.__evaluate_if(' '.join(line.split()[1:])) + self.__ifblocks.append([ _check, _check]) self.__ifconditions.append(' '.join(line.split()[1:])) # since in version <=0.7.7, it didn't handle #if it should be #elseifdef instead. # kept elseif with 2 elements for retro-compatibility (equivalent to #elseifdef). elif self.__is_directive(line, 'elseif') or \ self.__is_directive(line, 'elif'): - # do else - self.__ifblocks[-1] = not self.__ifblocks[-1] - # do if + _cur, _whole = self.__ifblocks[-1] if len(line.split()) == 2: #old behaviour - self.__ifblocks[-1] = self.__is_defined(line.split()[1]) + _check = self.__is_defined(line.split()[1]) else: #new behaviour - self.__ifblocks[-1] = self.__evaluate_if(' '.join(line.split()[1:])) - self.__ifconditions.append(' '.join(line.split()[1:])) + _check = self.__evaluate_if(' '.join(line.split()[1:])) + self.__ifblocks[-1]=[ not _whole and _check, _whole or _check ] + self.__ifconditions[-1]=' '.join(line.split()[1:]) elif self.__is_directive(line, 'elseifdef', 2): - # do else - self.__ifblocks[-1] = not self.__ifblocks[-1] - # do if - self.__ifblocks[-1]=self.__is_defined(line.split()[1]) - self.__ifconditions.append(' '.join(line.split()[1:])) + _cur, _whole = self.__ifblocks[-1] + _check = self.__is_defined(line.split()[1]) + self.__ifblocks[-1]=[ not _whole and _check, _whole or _check ] + self.__ifconditions[-1]=' '.join(line.split()[1:]) elif self.__is_directive(line, 'else', 1): - self.__ifblocks[-1] = not self.__ifblocks[-1] #opposite of last if + _cur, _whole = self.__ifblocks[-1] + self.__ifblocks[-1] = [not _whole, not _whole] #opposite of the whole if/elif block elif self.__is_directive(line, 'endififdef', 2): # do endif @@ -299,19 +307,17 @@ def lexer(self, line): except ValueError as VE: number = 1 - if len(self.__ifconditions) >= number: - for i in range(0, number): + try: + while number: self.__ifblocks.pop(-1) self.__ifconditions.pop(-1) - else: + number-=1 + except: if not self.quiet: print('Warning trying to remove more blocks than present', self.input, self.__linenum) - self.__ifblocks = [] - self.__ifconditions = [] else: - # unknown directive or comment # escapechar + space ==> comment # starts with #!/ ==> shebang # else print warning diff --git a/pypreprocessor/__main__.py b/pypreprocessor/__main__.py index 437f87b..0ecd8ee 100644 --- a/pypreprocessor/__main__.py +++ b/pypreprocessor/__main__.py @@ -8,30 +8,26 @@ python -m """+__package__+""" table.json --escape #@ -d NUM:4 ID:1 """, formatter_class=argparse.RawDescriptionHelpFormatter, - usage=' python -m '+__package__+' input [output] [-h] [-r] [-m] [-e ESCAPE] [-d [DEFINE ...]]', + usage=' python -m '+__package__+' input [output] [-h] [-r] [-m] [-e ESCAPE] [-o] [-q] [-d [DEFINE ...]]', ) -parser.add_argument("-r", "--run", help="Run on the fly", +parser.add_argument("-r", "--run", help="run on the fly", action='store_true', default=False) parser.add_argument("-m", "--removeMeta", help="remove meta lines from the output", action='store_true', default=False) parser.add_argument("-e", "--escape", help="define the escape sequence to use. Default is #") -parser.add_argument("-d", "--define", help="list of variable to define", nargs='*') +parser.add_argument("-d", "--define", help="list of constants to define", nargs='*', default=[]) parser.add_argument("-o", "--overload", help="overload variable definition in the file by those \ provided by --define", action='store_true', default=False) -parser.add_argument("-q", "--quiet", help="No warning on ununderstable directives and missign ending", +parser.add_argument("-q", "--quiet", help="No warning on not understood directives and missign ending", action='store_true', default=False) parser.add_argument("input", help="input file.") parser.add_argument("output", nargs='?', help="output file. Default is _out.") args = parser.parse_args() -p=preprocessor(inFile=args.input, mode=None, removeMeta=args.removeMeta, escapeChar=None, +p=preprocessor(inFile=args.input, defines=args.define, mode=None, removeMeta=args.removeMeta, escapeChar=None, run=args.run, resume=False, save=True, overload=args.overload, quiet=args.quiet) if args.output: p.define = args.output -if args.define: - for elem in args.define: - p.define(*elem.split(':')) - self.__reset_internal() # not very pythonic if args.escape: p.escape = args.escape From a7d0f16e3daab13a2a446839a1ba4104f8f5ef47 Mon Sep 17 00:00:00 2001 From: Laurent Pinson Date: Tue, 21 Dec 2021 16:52:38 +0100 Subject: [PATCH 13/13] added #error directive --- pypreprocessor/__init__.py | 6 ++++++ pypreprocessor/__main__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pypreprocessor/__init__.py b/pypreprocessor/__init__.py index 04542fe..4c5371e 100644 --- a/pypreprocessor/__init__.py +++ b/pypreprocessor/__init__.py @@ -317,6 +317,12 @@ def lexer(self, line): print('Warning trying to remove more blocks than present', self.input, self.__linenum) + elif self.__is_directive(line, 'error'): + if self.__validate_ifs(): + print('File: "' + self.input + '", line ' + str(self.__linenum + 1)) + print('Error directive reached') + sys.exit(1) + else: # escapechar + space ==> comment # starts with #!/ ==> shebang diff --git a/pypreprocessor/__main__.py b/pypreprocessor/__main__.py index 0ecd8ee..73235b2 100644 --- a/pypreprocessor/__main__.py +++ b/pypreprocessor/__main__.py @@ -27,7 +27,7 @@ p=preprocessor(inFile=args.input, defines=args.define, mode=None, removeMeta=args.removeMeta, escapeChar=None, run=args.run, resume=False, save=True, overload=args.overload, quiet=args.quiet) if args.output: - p.define = args.output + p.output = args.output if args.escape: p.escape = args.escape