Skip to content

Commit e8159df

Browse files
JackwensenJackwensen
authored
add control net support (#156)
还有一些问题需要讨论和确定 1. 为支持Control Net需要的升级： - huggingface官方的 [diffusers 0.14.0](https://github.com/huggingface/diffusers/releases/tag/v0.14.0) 支持了Control Net pipeline。所以我们要支持的话也需要安装0.14.0版本的diffusers `python3 -m pip install "transformers>=4.26" "diffusers[torch]==0.14.0"` - canny example 中为了检测Canny edge用到了 cv2，需要安装 `pip install opencv-contrib-python` 2. 可能还需要测试升级diffusers版本后对其他pipeline的影响 3. 第一次提PR不知还有什么步骤需要完善 ### Reference - https://huggingface.co/docs/diffusers/v0.14.0/en/api/pipelines/stable_diffusion/controlnet - huggingface/diffusers#2407
1 parent e469262 commit e8159df

11 files changed

+873
-30
lines changed

‎README.md

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ Please refer to this [wiki](https://github.com/Oneflow-Inc/diffusers/wiki/How-to
1111
## Quick demo
1212

1313
```
14-
python3 -m pip install "transformers>=4.26" "diffusers[torch]==0.12.1"
14+
python3 -m pip install "transformers>=4.26" "diffusers[torch]==0.14.0"
1515
python3 -m pip uninstall accelerate -y
1616
python3 -m pip install -U onediff
1717
python3 -m onediff.demo
@@ -28,7 +28,7 @@ OneFlow's main [repo](https://github.com/Oneflow-Inc/oneflow)
2828
```
2929
git clone https://github.com/Oneflow-Inc/diffusers.git onediff
3030
cd onediff
31-
python3 -m pip install "transformers>=4.26" "diffusers[torch]==0.12.1"
31+
python3 -m pip install "transformers>=4.26" "diffusers[torch]==0.14.0"
3232
python3 -m pip uninstall accelerate -y
3333
python3 -m pip install -e .
3434
```
@@ -45,7 +45,7 @@ python3 -m pip uninstall diffusers -y
4545
2. install transformers and diffusers
4646

4747
```
48-
python3 -m pip install "transformers>=4.26" "diffusers[torch]==0.12.1"
48+
python3 -m pip install "transformers>=4.26" "diffusers[torch]==0.14.0"
4949
python3 -m pip uninstall accelerate -y
5050
```
5151

‎examples/control_net_canny.py

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
import cv2
2+
from PIL import Image
3+
import numpy as np
4+
5+
from onediff import OneFlowStableDiffusionControlNetPipeline
6+
7+
import oneflow as flow
8+
flow.mock_torch.enable()
9+
10+
from diffusers.utils import load_image
11+
from diffusers import ControlNetModel
12+
13+
image = load_image(
14+
"http://hf.co/datasets/huggingface/documentation-images/resolve/main/diffusers/input_image_vermeer.png"
15+
)
16+
17+
image = np.array(image)
18+
19+
low_threshold = 100
20+
high_threshold = 200
21+
22+
image = cv2.Canny(image, low_threshold, high_threshold)
23+
image = image[:, :, None]
24+
image = np.concatenate([image, image, image], axis=2)
25+
canny_image = Image.fromarray(image)
26+
27+
controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-canny", torch_dtype=flow.float16)
28+
29+
pipe = OneFlowStableDiffusionControlNetPipeline.from_pretrained(
30+
"runwayml/stable-diffusion-v1-5", controlnet=controlnet, torch_dtype=flow.float16
31+
)
32+
33+
pipe.to("cuda")
34+
35+
generator = flow.manual_seed(0)
36+
37+
prompt = "disco dancer with colorful lights, best quality, extremely detailed"
38+
negative_prompt = "longbody, lowres, bad anatomy, bad hands, missing fingers, extra digit, fewer digits, cropped, worst quality, low quality"
39+
40+
41+
out_images = pipe(
42+
prompt = prompt, negative_prompt=negative_prompt, num_inference_steps=20, generator=generator, image=canny_image
43+
).images
44+
for i, image in enumerate(out_images):
45+
image.save(f"{prompt}-of-{i}.png")

‎examples/image_to_image.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
1-
import oneflow as flow
21
from PIL import Image
3-
flow.mock_torch.enable()
2+
43
from onediff import OneFlowStableDiffusionImg2ImgPipeline
4+
import oneflow as flow
5+
flow.mock_torch.enable()
56

67
pipe = OneFlowStableDiffusionImg2ImgPipeline.from_pretrained(
78
"stabilityai/stable-diffusion-2",
@@ -22,7 +23,6 @@
2223
image=img,
2324
guidance_scale=10,
2425
num_inference_steps=100,
25-
compile_unet=False,
2626
output_type="np",
2727
).images
2828
for i, image in enumerate(images):

‎examples/text_to_image.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
1-
import oneflow as flow
1+
from onediff import OneFlowStableDiffusionPipeline
22

3+
import oneflow as flow
34
flow.mock_torch.enable()
4-
from onediff import OneFlowStableDiffusionPipeline
55

66
pipe = OneFlowStableDiffusionPipeline.from_pretrained(
77
"CompVis/stable-diffusion-v1-4",

‎examples/text_to_image_alt.py

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
1-
import oneflow as flow
2-
3-
flow.mock_torch.enable()
41
from onediff import OneFlowAltDiffusionPipeline
52

3+
import oneflow as flow
4+
flow.mock_torch.enable()
65

76
pipe = OneFlowAltDiffusionPipeline.from_pretrained("BAAI/AltDiffusion-m9", torch_dtype=flow.float16)
87
pipe = pipe.to("cuda")

‎examples/text_to_image_dpmsolver.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
1-
import oneflow as flow
1+
from onediff import OneFlowStableDiffusionPipeline
22

3+
import oneflow as flow
34
flow.mock_torch.enable()
5+
46
from diffusers import DPMSolverMultistepScheduler
5-
from onediff import OneFlowStableDiffusionPipeline
67

78
model_id = "CompVis/stable-diffusion-v1-4"
89

‎examples/text_to_image_inpaint.py

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,14 @@
1-
import oneflow as flow
2-
flow.mock_torch.enable()
3-
4-
import PIL
5-
import requests
6-
from io import BytesIO
71
from onediff import OneFlowStableDiffusionInpaintPipeline
82

3+
from diffusers.utils import load_image
94

10-
def download_image(url):
11-
response = requests.get(url)
12-
return PIL.Image.open(BytesIO(response.content)).convert("RGB")
13-
5+
import oneflow as flow
6+
flow.mock_torch.enable()
147

158
img_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png"
169
mask_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png"
17-
init_image = download_image(img_url).resize((512, 512))
18-
mask_image = download_image(mask_url).resize((512, 512))
19-
10+
init_image = load_image(img_url).resize((512, 512))
11+
mask_image = load_image(mask_url).resize((512, 512))
2012
pipe = OneFlowStableDiffusionInpaintPipeline.from_pretrained(
2113
"runwayml/stable-diffusion-inpainting",
2214
torch_dtype=flow.float16,

‎examples/text_to_image_sd2.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
1+
from onediff import OneFlowStableDiffusionPipeline
2+
13
import oneflow as flow
24
flow.mock_torch.enable()
35

46
from diffusers import EulerDiscreteScheduler
5-
from onediff import OneFlowStableDiffusionPipeline
67

78
model_id = "stabilityai/stable-diffusion-2"
89
# Use the Euler scheduler here instead

‎src/onediff/__init__.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,4 +31,5 @@ def dummy_randn(*args, **kwargs):
3131
from .pipeline_stable_diffusion_inpaint_oneflow import (
3232
OneFlowStableDiffusionInpaintPipeline,
3333
)
34+
from .pipeline_stable_diffusion_controlnet_oneflow import OneFlowStableDiffusionControlNetPipeline
3435

0 commit comments

Comments
 (0)