🕺How To: Style and Pose a Character with InstantID + Controlnet

This workflow guides you through the process of building a poseable character creator glif that uses InstantID and Controlnet.

Video Tutorial

Check out and remix our example Glif here!

Step by Step

1. Click ‘Build’ to start a new project.

2. Add an image input block and set up a user prompt.

3. Add a second image input block and set up the user prompt.

4. Add a text input block and set up the user prompt.

5. Label the different user input blocks accordingly.

6. Add a text generation block to expand on the user text input.

7. Add a ComfyUI block and set it up with the provided JSON file.

Text Generation Block Settings

Input the below settings into the text generation block.

Prompt

{text input block}

Max Tokens

200

Model

claude-3.5-sonnet

System Prompt

You are helping to design cool classic 90s anime style posters with awesome protagonists. Based on the input, please create a short description of a character such as:

Input: Blue haired fire princess from an alien planet
Response: elegant princess figure with flaming eyes and a sphere of fire surrounding her

ComfyUI JSON

{
  "3": {
    "inputs": {
      "seed": 416440346149983,
      "steps": 30,
      "cfg": 4.5,
      "sampler_name": "ddpm",
      "scheduler": "karras",
      "denoise": 1,
      "model": [
        "60",
        0
      ],
      "positive": [
        "60",
        1
      ],
      "negative": [
        "60",
        2
      ],
      "latent_image": [
        "5",
        0
      ]
    },
    "class_type": "KSampler",
    "_meta": {
      "title": "KSampler"
    }
  },
  "4": {
    "inputs": {
      "ckpt_name": "sd_xl_base_1.0.safetensors"
    },
    "class_type": "CheckpointLoaderSimple",
    "_meta": {
      "title": "Load Checkpoint"
    }
  },
  "5": {
    "inputs": {
      "width": 1016,
      "height": 1016,
      "batch_size": 1
    },
    "class_type": "EmptyLatentImage",
    "_meta": {
      "title": "Empty Latent Image"
    }
  },
  "8": {
    "inputs": {
      "samples": [
        "3",
        0
      ],
      "vae": [
        "4",
        2
      ]
    },
    "class_type": "VAEDecode",
    "_meta": {
      "title": "VAE Decode"
    }
  },
  "11": {
    "inputs": {
      "instantid_file": "SDXL/ip-adapter.bin"
    },
    "class_type": "InstantIDModelLoader",
    "_meta": {
      "title": "Load InstantID Model"
    }
  },
  "13": {
    "inputs": {
      "image": "{person1}"
    },
    "class_type": "LoadImage",
    "_meta": {
      "title": "Load Image"
    }
  },
  "16": {
    "inputs": {
      "control_net_name": "SDXL/instantid/diffusion_pytorch_model.safetensors"
    },
    "class_type": "ControlNetLoader",
    "_meta": {
      "title": "Load ControlNet Model"
    }
  },
  "38": {
    "inputs": {
      "provider": "CPU"
    },
    "class_type": "InstantIDFaceAnalysis",
    "_meta": {
      "title": "InstantID Face Analysis"
    }
  },
  "39": {
    "inputs": {
      "text": "an anime character, {prompt}, natural textures, soft linework, stars, glitter, galaxy, fine linework, amazing illustration, soft brushes, watercolor",
      "clip": [
        "69",
        1
      ]
    },
    "class_type": "CLIPTextEncode",
    "_meta": {
      "title": "CLIP Text Encode (Prompt)"
    }
  },
  "40": {
    "inputs": {
      "text": "photograph, deformed, glitch, noisy, realistic, stock photo",
      "clip": [
        "69",
        1
      ]
    },
    "class_type": "CLIPTextEncode",
    "_meta": {
      "title": "CLIP Text Encode (Prompt)"
    }
  },
  "60": {
    "inputs": {
      "weight": 0.8,
      "start_at": 0,
      "end_at": 1,
      "instantid": [
        "11",
        0
      ],
      "insightface": [
        "38",
        0
      ],
      "control_net": [
        "16",
        0
      ],
      "image": [
        "13",
        0
      ],
      "model": [
        "69",
        0
      ],
      "positive": [
        "71",
        0
      ],
      "negative": [
        "71",
        1
      ],
      "image_kps": [
        "67",
        0
      ]
    },
    "class_type": "ApplyInstantID",
    "_meta": {
      "title": "Apply InstantID"
    }
  },
  "67": {
    "inputs": {
      "image": "{pose1}"
    },
    "class_type": "LoadImage",
    "_meta": {
      "title": "Load Image"
    }
  },
  "69": {
    "inputs": {
      "repo_id": "alvdansen/frosting_lane_redux",
      "subfolder": "",
      "filename": "araminta_k_frosting_lane_redux.safetensors",
      "strength_model": 1,
      "strength_clip": 1,
      "model": [
        "4",
        0
      ],
      "clip": [
        "4",
        1
      ]
    },
    "class_type": "HFHubLoraLoader",
    "_meta": {
      "title": "Load HF Lora"
    }
  },
  "70": {
    "inputs": {
      "filename_prefix": "ComfyUI",
      "images": [
        "8",
        0
      ]
    },
    "class_type": "SaveImage",
    "_meta": {
      "title": "Save Image"
    }
  },
  "71": {
    "inputs": {
      "strength": 0.75,
      "start_percent": 0,
      "end_percent": 1,
      "positive": [
        "39",
        0
      ],
      "negative": [
        "40",
        0
      ],
      "control_net": [
        "72",
        0
      ],
      "image": [
        "67",
        0
      ]
    },
    "class_type": "ControlNetApplyAdvanced",
    "_meta": {
      "title": "Apply ControlNet (Advanced)"
    }
  },
  "72": {
    "inputs": {
      "control_net_name": "mistoLine_fp16.safetensors"
    },
    "class_type": "ControlNetLoader",
    "_meta": {
      "title": "Load ControlNet Model"
    }
  }
}