diff --git a/assets/qwen/qwen-image-2.1-alpha-in1.png b/assets/qwen/qwen-image-2.1-alpha-in1.png new file mode 100644 index 0000000000..e65fcf0719 Binary files /dev/null and b/assets/qwen/qwen-image-2.1-alpha-in1.png differ diff --git a/assets/qwen/qwen-image-2.1-alpha-out1.png b/assets/qwen/qwen-image-2.1-alpha-out1.png new file mode 100644 index 0000000000..6841b7f787 Binary files /dev/null and b/assets/qwen/qwen-image-2.1-alpha-out1.png differ diff --git a/assets/qwen/qwen-image-2.1-alpha-out2.png b/assets/qwen/qwen-image-2.1-alpha-out2.png new file mode 100644 index 0000000000..b3db6c892c Binary files /dev/null and b/assets/qwen/qwen-image-2.1-alpha-out2.png differ diff --git a/docs/qwen_image_2.1.md b/docs/qwen_image_2.1.md index 08a50875ec..31008395ed 100644 --- a/docs/qwen_image_2.1.md +++ b/docs/qwen_image_2.1.md @@ -39,3 +39,21 @@ Pass the reference image with `-r` and describe the edit in `-p`. Vision weights ``` For multiple reference images, repeat `-r` in the desired order, for example `-r first.png -r second.png`. + +### Alpha channel + +This model supports alpha channel output. As the model determines whether to output a regular image or with transparency through the prompt, according to [official recommendation](https://github.com/QwenLM/Qwen-Image-2.1#transparent-image-generation-rgba), use the following prompt format for better results: + +> `This is an RGBA image with transparency. . The image has alpha channel and the background is transparent.` + +Since transparency is decided by the prompt rather than by the input or an explicit switch, the same format applies equally to editing, whether or not the reference image itself has an alpha channel. Note that alpha is kept only in `.png` and `.webp` outputs; saving as `.jpg` drops the transparency. + +Here are some examples ran with Q6_K quantization: +| Input | Prompt | Output | +| --- | --- | --- | +| ![Qwen Image 2.1 alpha input example 1](../assets/qwen/qwen-image-2.1-alpha-in1.png) | This is an RGBA image with transparency. Replace the text "BLOOM" with "Qwen Image 2.1", keeping the same font of the original text. The image has alpha channel and the background is transparent. | ![Qwen Image 2.1 alpha output example 1](../assets/qwen/qwen-image-2.1-alpha-out1.png) | +| ![Qwen Image 2.1 alpha input example 2](../assets/logo.png) | This is an RGBA image with transparency. Remove the background of the image, keeping only the text and cat. The image has alpha channel and the background is transparent. | ![Qwen Image 2.1 alpha output example 2](../assets/qwen/qwen-image-2.1-alpha-out2.png) | + +### Other features + +Other features of the model could be found on the [model card from QwenLM/Qwen-Image-2.1 repo](https://github.com/QwenLM/Qwen-Image-2.1), including 2 finetuned prompt rewriting Qwen3.5-9B model. diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp index eb96d9b5fe..1a0bb28ee1 100644 --- a/examples/cli/main.cpp +++ b/examples/cli/main.cpp @@ -357,7 +357,8 @@ bool load_images_from_dir(const std::string dir, LOG_VERBOSE("load image %zu from '%s'", images.size(), path.c_str()); int width = 0; int height = 0; - uint8_t* image_buffer = load_image_from_file(path.c_str(), width, height, expected_width, expected_height); + int loaded_channel = 0; + uint8_t* image_buffer = load_image_from_file(path.c_str(), width, height, loaded_channel, expected_width, expected_height); if (image_buffer == nullptr) { LOG_ERROR("load image from '%s' failed", path.c_str()); return false; @@ -365,7 +366,7 @@ bool load_images_from_dir(const std::string dir, images.emplace_back(sd_image_t{(uint32_t)width, (uint32_t)height, - 3, + (uint32_t)loaded_channel, image_buffer}); if (max_image_num > 0 && static_cast(images.size()) >= max_image_num) { @@ -781,7 +782,8 @@ int main(int argc, const char* argv[]) { }; if (gen_params.init_image_path.size() > 0) { - if (!load_image_and_update_size(gen_params.init_image_path, gen_params.init_image)) { + const bool native_init = cli_params.mode == IMG_GEN || cli_params.mode == ADETAILER; + if (!load_image_and_update_size(gen_params.init_image_path, gen_params.init_image, true, native_init ? 0 : 3)) { return 1; } } @@ -795,8 +797,8 @@ int main(int argc, const char* argv[]) { if (gen_params.ref_image_paths.size() > 0) { gen_params.ref_images.clear(); for (auto& path : gen_params.ref_image_paths) { - SDImageOwner ref_image({0, 0, 3, nullptr}); - if (!load_image_and_update_size(path, ref_image, false)) { + SDImageOwner ref_image({0, 0, 0, nullptr}); + if (!load_image_and_update_size(path, ref_image, false, 0)) { return 1; } gen_params.ref_images.push_back(std::move(ref_image)); diff --git a/examples/common/common.cpp b/examples/common/common.cpp index cc97f6694d..25f78d3b14 100644 --- a/examples/common/common.cpp +++ b/examples/common/common.cpp @@ -1848,20 +1848,22 @@ bool decode_base64_image(const std::string& encoded_input, return false; } - int decoded_width = 0; - int decoded_height = 0; - uint8_t* raw_data = load_image_from_memory(reinterpret_cast(image_bytes.data()), - static_cast(image_bytes.size()), - decoded_width, - decoded_height, - expected_width, - expected_height, - target_channels); + int decoded_width = 0; + int decoded_height = 0; + int resolved_channel = target_channels; + uint8_t* raw_data = load_image_from_memory(reinterpret_cast(image_bytes.data()), + static_cast(image_bytes.size()), + decoded_width, + decoded_height, + resolved_channel, + expected_width, + expected_height, + target_channels); if (raw_data == nullptr) { return false; } - out_image.reset({(uint32_t)decoded_width, (uint32_t)decoded_height, (uint32_t)target_channels, raw_data}); + out_image.reset({(uint32_t)decoded_width, (uint32_t)decoded_height, (uint32_t)resolved_channel, raw_data}); return true; } @@ -2215,7 +2217,7 @@ bool SDGenerationParams::from_json_str( LOG_ERROR("invalid lora"); return false; } - if (!parse_image_json_field(j, "init_image", 3, width, height, init_image)) { + if (!parse_image_json_field(j, "init_image", 0, width, height, init_image)) { LOG_ERROR("invalid init_image"); return false; } @@ -2225,7 +2227,7 @@ bool SDGenerationParams::from_json_str( } if (!parse_image_array_json_field(j, "ref_images", - 3, + 0, auto_resize_ref_image ? width : 0, auto_resize_ref_image ? height : 0, ref_images)) { diff --git a/examples/common/media_io.cpp b/examples/common/media_io.cpp index 345b6f1523..ba0e8c5b29 100644 --- a/examples/common/media_io.cpp +++ b/examples/common/media_io.cpp @@ -261,6 +261,10 @@ uint8_t* decode_webp_image_to_buffer(const uint8_t* data, height = features.height; source_channel_count = features.has_alpha ? 4 : 3; + if (expected_channel == 0) { + expected_channel = source_channel_count; + } + const size_t pixel_count = static_cast(width) * static_cast(height); if (expected_channel == 1) { @@ -481,7 +485,8 @@ uint8_t* load_image_common(bool from_memory, int& height, int expected_width, int expected_height, - int expected_channel) { + int expected_channel, + int& out_channel) { const char* image_path; FreeUniquePtr image_buffer; int source_channel_count = 0; @@ -538,6 +543,32 @@ uint8_t* load_image_common(bool from_memory, LOG_ERROR("load image from '%s' failed", image_path); return nullptr; } + if (expected_channel == 0) { + expected_channel = source_channel_count == 2 ? 4 : (source_channel_count == 1 ? 3 : source_channel_count); + if (expected_channel != source_channel_count) { + FreeUniquePtr promoted((uint8_t*)malloc((size_t)width * height * expected_channel)); + if (promoted == nullptr) { + LOG_ERROR("error: allocate memory for channel promotion, image_path = %s", image_path); + return nullptr; + } + const size_t pixel_count = (size_t)width * (size_t)height; + for (size_t i = 0; i < pixel_count; ++i) { + if (source_channel_count == 1) { + promoted.get()[i * 3 + 0] = image_buffer.get()[i]; + promoted.get()[i * 3 + 1] = image_buffer.get()[i]; + promoted.get()[i * 3 + 2] = image_buffer.get()[i]; + } else { + promoted.get()[i * 4 + 0] = image_buffer.get()[i * 2]; + promoted.get()[i * 4 + 1] = image_buffer.get()[i * 2]; + promoted.get()[i * 4 + 2] = image_buffer.get()[i * 2]; + promoted.get()[i * 4 + 3] = image_buffer.get()[i * 2 + 1]; + } + } + image_buffer = std::move(promoted); + source_channel_count = expected_channel; + } + } + // stb reports the source channel count even when it converts the output. if (source_channel_count < expected_channel) { fprintf(stderr, "the number of channels for the input image must be >= %d," @@ -597,7 +628,7 @@ uint8_t* load_image_common(bool from_memory, } stbir_resize(image_buffer.get(), width, height, 0, resized_image_buffer.get(), expected_width, expected_height, 0, STBIR_TYPE_UINT8, - expected_channel, STBIR_ALPHA_CHANNEL_NONE, 0, + expected_channel, expected_channel == 4 ? 3 : STBIR_ALPHA_CHANNEL_NONE, 0, STBIR_EDGE_CLAMP, STBIR_EDGE_CLAMP, STBIR_FILTER_BOX, STBIR_FILTER_BOX, STBIR_COLORSPACE_SRGB, nullptr); @@ -605,6 +636,7 @@ uint8_t* load_image_common(bool from_memory, height = expected_height; image_buffer = std::move(resized_image_buffer); } + out_channel = expected_channel; return image_buffer.release(); } @@ -777,10 +809,11 @@ bool write_image_to_file(const std::string& path, uint8_t* load_image_from_file(const char* image_path, int& width, int& height, + int& out_channel, int expected_width, int expected_height, int expected_channel) { - return load_image_common(false, image_path, 0, width, height, expected_width, expected_height, expected_channel); + return load_image_common(false, image_path, 0, width, height, expected_width, expected_height, expected_channel, out_channel); } bool load_sd_image_from_file(sd_image_t* image, @@ -790,13 +823,14 @@ bool load_sd_image_from_file(sd_image_t* image, int expected_channel) { int width; int height; - image->data = load_image_common(false, image_path, 0, width, height, expected_width, expected_height, expected_channel); + int resolved_channel = expected_channel; + image->data = load_image_common(false, image_path, 0, width, height, expected_width, expected_height, expected_channel, resolved_channel); if (image->data == nullptr) { return false; } image->width = width; image->height = height; - image->channel = expected_channel; + image->channel = resolved_channel; return true; } @@ -804,10 +838,11 @@ uint8_t* load_image_from_memory(const char* image_bytes, int len, int& width, int& height, + int& out_channel, int expected_width, int expected_height, int expected_channel) { - return load_image_common(true, image_bytes, len, width, height, expected_width, expected_height, expected_channel); + return load_image_common(true, image_bytes, len, width, height, expected_width, expected_height, expected_channel, out_channel); } static void append_avi_metadata(std::vector& data, const std::string& parameters) { diff --git a/examples/common/media_io.h b/examples/common/media_io.h index 6adbe98758..b2b56eccde 100644 --- a/examples/common/media_io.h +++ b/examples/common/media_io.h @@ -32,9 +32,12 @@ bool write_image_to_file(const std::string& path, const std::string& parameters = "", int quality = 90); +// expected_channel == 0 preserves native channels (grayscale -> RGB, gray+alpha -> RGBA). +// out_channel receives the output channel count. uint8_t* load_image_from_file(const char* image_path, int& width, int& height, + int& out_channel, int expected_width = 0, int expected_height = 0, int expected_channel = 3); @@ -49,6 +52,7 @@ uint8_t* load_image_from_memory(const char* image_bytes, int len, int& width, int& height, + int& out_channel, int expected_width = 0, int expected_height = 0, int expected_channel = 3); diff --git a/examples/server/api.md b/examples/server/api.md index 3ec9a1da8f..ffcc96e2b4 100644 --- a/examples/server/api.md +++ b/examples/server/api.md @@ -735,12 +735,15 @@ Any image field accepts: Channel expectations: -- `init_image`: 3 channels -- `ref_images[]`: 3 channels +- `init_image`: native channels (3 or 4); alpha is preserved and applied per model +- `ref_images[]`: native channels (3 or 4); alpha is preserved and applied per model - `control_image`: 3 channels - `ip_adapter_image`: 3 channels - `mask_image`: 1 channel +Models that support RGBA (e.g. Qwen-Image 2.1) use the alpha channel of `init_image` +and `ref_images[]`. RGB-only models drop it, so sending RGBA is safe for every model. + If omitted or null: - single-image fields map to an empty `sd_image_t` diff --git a/examples/server/routes_openai.cpp b/examples/server/routes_openai.cpp index ccde2baef0..de36091ca5 100644 --- a/examples/server/routes_openai.cpp +++ b/examples/server/routes_openai.cpp @@ -158,19 +158,20 @@ static bool build_openai_edit_request(const httplib::Request& req, request.gen_params.batch_count = n; for (auto& bytes : images_bytes) { - int img_w = 0; - int img_h = 0; - uint8_t* raw_pixels = load_image_from_memory( - reinterpret_cast(bytes.data()), - static_cast(bytes.size()), - img_w, img_h, - 0, 0, 3); + int img_w = 0; + int img_h = 0; + int resolved_channel = 0; + uint8_t* raw_pixels = load_image_from_memory( + reinterpret_cast(bytes.data()), + static_cast(bytes.size()), + img_w, img_h, resolved_channel, + 0, 0, 0); if (raw_pixels == nullptr) { continue; } const bool is_first_ref_image = request.gen_params.ref_images.empty(); - SDImageOwner image_owner({(uint32_t)img_w, (uint32_t)img_h, 3, raw_pixels}); + SDImageOwner image_owner({(uint32_t)img_w, (uint32_t)img_h, (uint32_t)resolved_channel, raw_pixels}); request.gen_params.set_width_and_height_if_unset(image_owner.get().width, image_owner.get().height); if (is_first_ref_image) { @@ -181,15 +182,16 @@ static bool build_openai_edit_request(const httplib::Request& req, init_h = request.gen_params.height; } - int init_img_w = 0; - int init_img_h = 0; - uint8_t* init_pixels = load_image_from_memory( - reinterpret_cast(bytes.data()), - static_cast(bytes.size()), - init_img_w, init_img_h, - init_w, init_h, 3); + int init_img_w = 0; + int init_img_h = 0; + int init_resolved_channel = 0; + uint8_t* init_pixels = load_image_from_memory( + reinterpret_cast(bytes.data()), + static_cast(bytes.size()), + init_img_w, init_img_h, init_resolved_channel, + init_w, init_h, 0); if (init_pixels != nullptr) { - request.gen_params.init_image.reset({(uint32_t)init_img_w, (uint32_t)init_img_h, 3, init_pixels}); + request.gen_params.init_image.reset({(uint32_t)init_img_w, (uint32_t)init_img_h, (uint32_t)init_resolved_channel, init_pixels}); } } @@ -203,13 +205,14 @@ static bool build_openai_edit_request(const httplib::Request& req, expected_width = request.gen_params.width; expected_height = request.gen_params.height; } - int mask_w = 0; - int mask_h = 0; + int mask_w = 0; + int mask_h = 0; + int mask_channel = 0; uint8_t* mask_raw = load_image_from_memory( reinterpret_cast(mask_bytes.data()), static_cast(mask_bytes.size()), - mask_w, mask_h, + mask_w, mask_h, mask_channel, expected_width, expected_height, 1); request.gen_params.mask_image.reset({(uint32_t)mask_w, (uint32_t)mask_h, 1, mask_raw}); const sd_image_t& mask_image = request.gen_params.mask_image.get(); diff --git a/examples/server/routes_sdapi.cpp b/examples/server/routes_sdapi.cpp index f6bd094970..1966a08ff9 100644 --- a/examples/server/routes_sdapi.cpp +++ b/examples/server/routes_sdapi.cpp @@ -199,7 +199,7 @@ static bool build_sdapi_img_gen_request(const json& j, if (j.contains("init_images") && j["init_images"].is_array() && !j["init_images"].empty()) { if (decode_base64_image(j["init_images"][0].get(), - 3, + 0, expected_width, expected_height, request.gen_params.init_image)) { @@ -243,7 +243,7 @@ static bool build_sdapi_img_gen_request(const json& j, } SDImageOwner image_owner; if (decode_base64_image(extra_image.get(), - 3, + 0, request.gen_params.auto_resize_ref_image && request.gen_params.width_and_height_are_set() ? request.gen_params.width : 0, diff --git a/src/pipeline/video.cpp b/src/pipeline/video.cpp index 503afa530a..3127ecb635 100644 --- a/src/pipeline/video.cpp +++ b/src/pipeline/video.cpp @@ -470,11 +470,15 @@ namespace sd::pipeline { sd::Tensor end_image; if (sd_vid_gen_params->init_image.data) { - start_image = sd_image_to_tensor(sd_vid_gen_params->init_image, request->width, request->height); + start_image = ensure_image_tensor_channels( + sd_image_to_tensor(sd_vid_gen_params->init_image, request->width, request->height), + sd->get_image_channels()); } if (sd_vid_gen_params->end_image.data) { - end_image = sd_image_to_tensor(sd_vid_gen_params->end_image, request->width, request->height); + end_image = ensure_image_tensor_channels( + sd_image_to_tensor(sd_vid_gen_params->end_image, request->width, request->height), + sd->get_image_channels()); } if (sd_version_is_minimax_h3(sd->version)) { @@ -1416,7 +1420,9 @@ namespace sd::pipeline { sd::Tensor video_mask = make_ltxav_video_denoise_mask(video_latent, 1.f); if (sd_vid_gen_params->init_image.data != nullptr) { - sd::Tensor start_image = sd_image_to_tensor(sd_vid_gen_params->init_image, image_width, image_height); + sd::Tensor start_image = ensure_image_tensor_channels( + sd_image_to_tensor(sd_vid_gen_params->init_image, image_width, image_height), + sd->get_image_channels()); if (!apply_ltxav_condition_image_by_latent_index(sd, start_image, &video_latent, @@ -1429,7 +1435,9 @@ namespace sd::pipeline { } if (sd_vid_gen_params->end_image.data != nullptr) { - sd::Tensor end_image = sd_image_to_tensor(sd_vid_gen_params->end_image, image_width, image_height); + sd::Tensor end_image = ensure_image_tensor_channels( + sd_image_to_tensor(sd_vid_gen_params->end_image, image_width, image_height), + sd->get_image_channels()); sd::Tensor end_image_latent = encode_ltxav_condition_image(sd, end_image, "end"); if (end_image_latent.empty()) { return false;