[Windows] Fix build issues using Clang-CL on Windows, add CI

Author: GregoryComer

.github/workflows/pull.yml

@@ -34,3 +34,44 @@ jobs:
 
         # Run tests
         pytest
+ 
+  windows:
+    uses: pytorch/test-infra/.github/workflows/windows_job.yml@main
+    with:
+      runner: windows.4xlarge
+      submodules: 'recursive'
+      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      script: |
+        conda init powershell
+        powershell -Command "& {
+          Set-PSDebug -Trace 1
+          \$ErrorActionPreference = 'Stop'
+          \$PSNativeCommandUseErrorActionPreference = \$true
+          
+          cmake -DCMAKE_BUILD_TYPE=Debug test -Bbuild/test -T ClangCL
+          cmake --build build/test -j9 --config Debug
+          if (\$LASTEXITCODE -ne 0) {
+            Write-Host "Build was not successful. Exit code: \$LASTEXITCODE."
+            exit \$LASTEXITCODE
+          }
+
+          Push-Location build/test
+          ctest
+          if (\$LASTEXITCODE -ne 0) {
+            Write-Host "Unit tests were not successful. Exit code: \$LASTEXITCODE."
+            exit \$LASTEXITCODE
+          }
+          Pop-Location
+
+          # Install tokenizers
+          pip install . -v
+          pip install pytest blobfile transformers>=4.53.1
+
+          # Run tests
+          pytest
+          if (\$LASTEXITCODE -ne 0) {
+            Write-Host "Python tests were not successful. Exit code: \$LASTEXITCODE."
+            Start-Sleep -Seconds 600 # Debug - keep alive to give time to SSH
+            exit \$LASTEXITCODE
+          }
+        }"

.gitignore

@@ -34,3 +34,7 @@ pip-out/
 *~
 .~lock.*
 *.idea
+
+*.so
+*.dylib
+*.pyd

CMakeLists.txt

@@ -28,7 +28,9 @@ include(CMakePackageConfigHelpers)
 include(Utils.cmake)
 
 # Ignore weak attribute warning
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-attributes")
+if(NOT MSVC)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-attributes")
+endif()
 
 set(ABSL_ENABLE_INSTALL ON)
 set(ABSL_PROPAGATE_CXX_STD ON)

include/pytorch/tokenizers/tiktoken.h

@@ -23,6 +23,12 @@
 #include <pytorch/tokenizers/result.h>
 #include <pytorch/tokenizers/tokenizer.h>
 
+#ifdef _WIN32
+// ssize_t isn't available on Windows. Alias it to the Windows SSIZE_T value.
+#include <BaseTsd.h>
+typedef SSIZE_T ssize_t;
+#endif
+
 namespace tokenizers {
 
 static constexpr int32_t kSpecialTokensSize = 256;

setup.py

@@ -30,9 +30,6 @@ class CMakeBuild(build_ext):
     def build_extension(self, ext):  # noqa C901
         extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name)))
 
-        # Ensure the extension goes into the pytorch_tokenizers package directory
-        extdir = os.path.join(extdir, "pytorch_tokenizers")
-
         # Required for auto-detection & inclusion of auxiliary "native" libs
         if not extdir.endswith(os.path.sep):
             extdir += os.path.sep
@@ -55,6 +52,10 @@ def build_extension(self, ext):  # noqa C901
         ]
         build_args = ["--target", "pytorch_tokenizers_cpp"]
 
+        # Use Clang for Windows builds.
+        if sys.platform == "win32":
+            cmake_args += ["-T ClangCL"]
+
         # Adding CMake arguments set as environment variable
         # (needed e.g. to build for ARM OSX on conda-forge)
         if "CMAKE_ARGS" in os.environ:
@@ -124,15 +125,14 @@ def build_extension(self, ext):  # noqa C901
             ["cmake", "--build", "."] + build_args, cwd=build_temp, check=True
         )
 
-
 setup(
     name="pytorch-tokenizers",
     version="0.1.0",
     long_description=long_description,
     long_description_content_type="text/markdown",
     url="https://github.com/meta-pytorch/tokenizers",
     packages=find_packages(),
-    ext_modules=[CMakeExtension("pytorch_tokenizers_cpp")],
+    ext_modules=[CMakeExtension("pytorch_tokenizers.pytorch_tokenizers_cpp")],
     cmdclass={"build_ext": CMakeBuild},
     zip_safe=False,
     python_requires=">=3.10",

src/hf_tokenizer.cpp

@@ -34,14 +34,14 @@ Error HFTokenizer::load(const std::string& path) {
   std::string model_config_json = "";
   if (fs::is_directory(path)) {
     const fs::path root(path);
-    model_json = root / "tokenizer.json";
+    model_json = (root / "tokenizer.json").string();
     if (!fs::exists(model_json)) {
       TK_LOG(Info, "no tokenizer.json found in %s", path.c_str());
       return Error::LoadFailure;
     }
     const auto model_config_json_path = root / "tokenizer_config.json";
     if (fs::exists(model_config_json_path)) {
-      model_config_json = model_config_json_path;
+      model_config_json = model_config_json_path.string();
     }
   }
 

test/test_tiktoken.cpp

@@ -117,7 +117,7 @@ TEST_F(TiktokenTest, TokenizerDecodeOutOfRangeFails) {
 TEST_F(TiktokenTest, ConstructionWithInvalidBOSIndex) {
   // gtest death test doesn't work on iOS:
   // https://github.com/google/googletest/issues/2834
-#if !GTEST_OS_IOS
+#if !GTEST_OS_IOS && !GTEST_OS_WINDOWS
   EXPECT_EXIT(
       std::make_unique<Tiktoken>(
           std::make_unique<std::vector<std::string>>(
@@ -132,7 +132,7 @@ TEST_F(TiktokenTest, ConstructionWithInvalidBOSIndex) {
 TEST_F(TiktokenTest, ConstructionWithInvalidEOSIndex) {
   // gtest death test doesn't work on iOS:
   // https://github.com/google/googletest/issues/2834
-#if !GTEST_OS_IOS
+#if !GTEST_OS_IOS && !GTEST_OS_WINDOWS
   EXPECT_EXIT(
       std::make_unique<Tiktoken>(
           std::make_unique<std::vector<std::string>>(