Add mlx-vlm

Author: mudler

Makefile

@@ -369,6 +369,10 @@ backends/mlx: build
 	BACKEND=mlx BUILD_TYPE=mps $(MAKE) build-darwin-python-backend
 	./local-ai backends install "ocifile://$(abspath ./backend-images/mlx.tar)"
 
+backends/mlx-vlm: build
+	BACKEND=mlx-vlm BUILD_TYPE=mps bash ./scripts/build/python-darwin.sh
+	./local-ai backends install "ocifile://$(abspath ./backend-images/mlx-vlm.tar)"
+
 backend-images:
 	mkdir -p backend-images
 

backend/index.yaml

@@ -142,6 +142,23 @@
     - text-to-text
     - LLM
     - MLX
+- &mlx-vlm
+  name: "mlx-vlm"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-mlx-vlm"
+  icon: https://avatars.githubusercontent.com/u/102832242?s=200&v=4
+  urls:
+    - https://github.com/ml-explore/mlx-vlm
+  mirrors:
+    - localai/localai-backends:latest-metal-darwin-arm64-mlx-vlm
+  license: MIT
+  description: |
+      Run Vision-Language Models with MLX
+  tags:
+    - text-to-text
+    - multimodal
+    - vision-language
+    - LLM
+    - MLX
 - &rerankers
   name: "rerankers"
   alias: "rerankers"
@@ -391,6 +408,11 @@
   uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-mlx"
   mirrors:
     - localai/localai-backends:master-metal-darwin-arm64-mlx
+- !!merge <<: *mlx-vlm
+  name: "mlx-vlm-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-mlx-vlm"
+  mirrors:
+    - localai/localai-backends:master-metal-darwin-arm64-mlx-vlm
 - !!merge <<: *kitten-tts
   name: "kitten-tts-development"
   uri: "quay.io/go-skynet/local-ai-backends:master-kitten-tts"

backend/python/mlx-vlm/Makefile

@@ -0,0 +1,23 @@
+.PHONY: mlx-vlm
+mlx-vlm:
+	bash install.sh
+
+.PHONY: run
+run: mlx-vlm
+	@echo "Running mlx-vlm..."
+	bash run.sh
+	@echo "mlx run."
+
+.PHONY: test
+test: mlx-vlm
+	@echo "Testing mlx-vlm..."
+	bash test.sh
+	@echo "mlx tested."
+
+.PHONY: protogen-clean
+protogen-clean:
+	$(RM) backend_pb2_grpc.py backend_pb2.py
+
+.PHONY: clean
+clean: protogen-clean
+	rm -rf venv __pycache__