Merge pull request #1463 from Thirumalai-Shaktivel/fix_carriage_return

Enable `run_tests` in Windows CI

Author: certik

.gitattributes

@@ -0,0 +1 @@
+**/*.py -text

ci/test.xsh

@@ -15,7 +15,9 @@ src/bin/lpython --show-ast tests/doconcurrentloop_01.py
 src/bin/lpython --show-asr tests/doconcurrentloop_01.py
 src/bin/lpython --show-cpp tests/doconcurrentloop_01.py
 
-if $WIN != "1":
+if $WIN == "1":
+    python run_tests.py --skip-run-with-dbg --no-color
+else:
     python run_tests.py
     cd integration_tests
     python run_tests.py -j16 -b llvm cpython c wasm

run_tests.py

@@ -10,7 +10,7 @@
 
 
 def single_test(test, verbose, no_llvm, skip_run_with_dbg, update_reference,
-                specific_backends=None, excluded_backends=None):
+                no_color, specific_backends=None, excluded_backends=None):
     filename = test["filename"]
     def is_included(backend):
          return test.get(backend, False) \
@@ -36,7 +36,10 @@ def is_included(backend):
     if pass_ and (pass_ not in ["do_loops", "global_stmts"] and
                   pass_ not in optimization_passes):
         raise Exception(f"Unknown pass: {pass_}")
-    log.debug(f"{color(style.bold)} START TEST: {color(style.reset)} {filename}")
+    if no_color:
+        log.debug(f" START TEST: {filename}")
+    else:
+        log.debug(f"{color(style.bold)} START TEST: {color(style.reset)} {filename}")
 
     extra_args = f"--no-error-banner {show_verbose}"
 

src/bin/lpython.cpp

@@ -1067,7 +1067,7 @@ int link_executable(const std::vector<std::string> &infiles,
             for (auto &s : infiles) {
                 cmd += s + " ";
             }
-            cmd += runtime_library_dir + "\\lpython_runtime_static.lib";
+            cmd += runtime_library_dir + "\\lpython_runtime_static.lib > NUL";
             int err = system(cmd.c_str());
             if (err) {
                 std::cout << "The command '" + cmd + "' failed." << std::endl;

src/libasr/compiler_tester/tester.py

@@ -29,6 +29,7 @@
 SRC_DIR = os.path.dirname(LIBASR_DIR)
 ROOT_DIR = os.path.dirname(SRC_DIR)
 
+no_color = False
 
 class RunException(Exception):
     pass
@@ -329,7 +330,10 @@ def run_test(testname, basename, cmd, infile, update_reference=False,
         raise RunException(
             "Testing with reference output failed." +
             full_err_str)
-    log.debug(s + " " + check())
+    if no_color:
+        log.debug(s + " PASS")
+    else:
+        log.debug(s + " " + check())
 
 
 def tester_main(compiler, single_test):
@@ -358,6 +362,8 @@ def tester_main(compiler, single_test):
                         help="Skip runtime tests with debugging information enabled")
     parser.add_argument("-s", "--sequential", action="store_true",
                         help="Run all tests sequentially")
+    parser.add_argument("--no-color", action="store_true",
+                    help="Turn off colored tests output")
     args = parser.parse_args()
     update_reference = args.update
     list_tests = args.list
@@ -374,6 +380,8 @@ def tester_main(compiler, single_test):
     verbose = args.verbose
     no_llvm = args.no_llvm
     skip_run_with_dbg = args.skip_run_with_dbg
+    global no_color
+    no_color = args.no_color
 
     # So that the tests find the `lcompiler` executable
     os.environ["PATH"] = os.path.join(SRC_DIR, "bin") \
@@ -401,7 +409,8 @@ def tester_main(compiler, single_test):
                         excluded_backends=excluded_backends,
                         verbose=verbose,
                         no_llvm=no_llvm,
-                        skip_run_with_dbg=skip_run_with_dbg)
+                        skip_run_with_dbg=skip_run_with_dbg,
+                        no_color=no_color)
     # run in parallel
     else:
         single_tester_partial_args = partial(
@@ -411,7 +420,8 @@ def tester_main(compiler, single_test):
             excluded_backends=excluded_backends,
             verbose=verbose,
             no_llvm=no_llvm,
-            skip_run_with_dbg=skip_run_with_dbg)
+            skip_run_with_dbg=skip_run_with_dbg,
+            no_color=no_color)
         with ThreadPoolExecutor() as ex:
             futures = ex.map(single_tester_partial_args, filtered_tests)
             for f in futures:
@@ -423,5 +433,9 @@ def tester_main(compiler, single_test):
     if update_reference:
         log.info("Test references updated.")
     else:
-        log.info(
-            f"{(color(fg.green) + color(style.bold))}TESTS PASSED{color(fg.reset) + color(style.reset)}")
+        if no_color:
+            log.info("TESTS PASSED")
+        else:
+            log.info(
+                f"{(color(fg.green) + color(style.bold))}TESTS PASSED"
+                f"{color(fg.reset) + color(style.reset)}")

src/lpython/parser/tokenizer.re

@@ -264,8 +264,8 @@ int Tokenizer::lex(Allocator &al, YYSTYPE &yylval, Location &loc, diag::Diagnost
             re2c:define:YYCTYPE = "unsigned char";
 
             end = "\x00";
-            whitespace = [ \t\v\r]+;
-            newline = "\n";
+            whitespace = [ \t\v]+;
+            newline = "\n" | "\r\n";
             digit = [0-9];
             int_oct = "0"[oO]([0-7] | "_" [0-7])+;
             int_bin = "0"[bB]([01] | "_" [01])+;