[pull] master from mudler:master

backend/backend.proto

@@ -301,7 +301,6 @@ message TranscriptSegment {
 message GenerateImageRequest {
   int32 height = 1;
   int32 width = 2;
-  int32 mode = 3;
   int32 step = 4;
   int32 seed = 5;
   string positive_prompt = 6;

backend/cpp/llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-LLAMA_VERSION?=0f89d2ecf14270f45f43c442e90ae433fd82dab1
+LLAMA_VERSION?=13814eb370d2f0b70e1830cc577b6155b17aee47
 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp
 
 CMAKE_ARGS?=

backend/go/whisper/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # whisper.cpp version
 WHISPER_REPO?=https://github.com/ggml-org/whisper.cpp
-WHISPER_CPP_VERSION?=6114e692136bea917dc88a5eb2e532c3d133d963
+WHISPER_CPP_VERSION?=e9898ddfb908ffaa7026c66852a023889a5a7202
 SO_TARGET?=libgowhisper.so
 
 CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF

core/backend/image.go

@@ -7,7 +7,7 @@ import (
 	model "github.com/mudler/LocalAI/pkg/model"
 )
 
-func ImageGeneration(height, width, mode, step, seed int, positive_prompt, negative_prompt, src, dst string, loader *model.ModelLoader, modelConfig config.ModelConfig, appConfig *config.ApplicationConfig, refImages []string) (func() error, error) {
+func ImageGeneration(height, width, step, seed int, positive_prompt, negative_prompt, src, dst string, loader *model.ModelLoader, modelConfig config.ModelConfig, appConfig *config.ApplicationConfig, refImages []string) (func() error, error) {
 
 	opts := ModelOptions(modelConfig, appConfig)
 	inferenceModel, err := loader.Load(
@@ -23,7 +23,6 @@ func ImageGeneration(height, width, mode, step, seed int, positive_prompt, negat
 			&proto.GenerateImageRequest{
 				Height:           int32(height),
 				Width:            int32(width),
-				Mode:             int32(mode),
 				Step:             int32(step),
 				Seed:             int32(seed),
 				CLIPSkip:         int32(modelConfig.Diffusers.ClipSkip),

core/http/endpoints/openai/image.go

@@ -157,16 +157,11 @@ func ImageEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi
 					negative_prompt = prompts[1]
 				}
 
-				mode := 0
 				step := config.Step
 				if step == 0 {
 					step = 15
 				}
 
-				if input.Mode != 0 {
-					mode = input.Mode
-				}
-
 				if input.Step != 0 {
 					step = input.Step
 				}
@@ -197,7 +192,7 @@ func ImageEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfi
 					inputSrc = inputImages[0]
 				}
 
-				fn, err := backend.ImageGeneration(height, width, mode, step, *config.Seed, positive_prompt, negative_prompt, inputSrc, output, ml, *config, appConfig, refImages)
+				fn, err := backend.ImageGeneration(height, width, step, *config.Seed, positive_prompt, negative_prompt, inputSrc, output, ml, *config, appConfig, refImages)
 				if err != nil {
 					return err
 				}

core/http/endpoints/openai/inpainting.go

@@ -231,7 +231,7 @@ func InpaintingEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app
 		// Note: ImageGenerationFunc will call into the loaded model's GenerateImage which expects src JSON
 		// Also pass ref images (orig + mask) so backends that support ref images can use them.
 		refImages := []string{origRef, maskRef}
-		fn, err := backend.ImageGenerationFunc(height, width, 0, steps, 0, prompt, "", jsonPath, dst, ml, *cfg, appConfig, refImages)
+		fn, err := backend.ImageGenerationFunc(height, width, steps, 0, prompt, "", jsonPath, dst, ml, *cfg, appConfig, refImages)
 		if err != nil {
 			return err
 		}

core/http/endpoints/openai/inpainting_test.go

@@ -10,9 +10,9 @@ import (
 	"testing"
 
 	"github.com/labstack/echo/v4"
-	"github.com/mudler/LocalAI/core/http/middleware"
 	"github.com/mudler/LocalAI/core/backend"
 	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/http/middleware"
 	model "github.com/mudler/LocalAI/pkg/model"
 	"github.com/stretchr/testify/require"
 )
@@ -58,7 +58,7 @@ func TestInpainting_HappyPath(t *testing.T) {
 
 	// stub the backend.ImageGenerationFunc
 	orig := backend.ImageGenerationFunc
-	backend.ImageGenerationFunc = func(height, width, mode, step, seed int, positive_prompt, negative_prompt, src, dst string, loader *model.ModelLoader, modelConfig config.ModelConfig, appConfig *config.ApplicationConfig, refImages []string) (func() error, error) {
+	backend.ImageGenerationFunc = func(height, width, step, seed int, positive_prompt, negative_prompt, src, dst string, loader *model.ModelLoader, modelConfig config.ModelConfig, appConfig *config.ApplicationConfig, refImages []string) (func() error, error) {
 		fn := func() error {
 			// write a fake png file to dst
 			return os.WriteFile(dst, []byte("PNGDATA"), 0644)

core/http/static/image.js

@@ -1,61 +1,255 @@
+// Helper function to convert file to base64
+function fileToBase64(file) {
+  return new Promise((resolve, reject) => {
+    const reader = new FileReader();
+    reader.onload = () => {
+      // Remove data:image/...;base64, prefix if present
+      const base64 = reader.result.split(',')[1] || reader.result;
+      resolve(base64);
+    };
+    reader.onerror = reject;
+    reader.readAsDataURL(file);
+  });
+}
+
+// Helper function to read multiple files
+async function filesToBase64Array(fileList) {
+  const base64Array = [];
+  for (let i = 0; i < fileList.length; i++) {
+    const base64 = await fileToBase64(fileList[i]);
+    base64Array.push(base64);
+  }
+  return base64Array;
+}
+
 function genImage(event) {
   event.preventDefault();
-  const input = document.getElementById("input").value;
-
-  promptDallE(input);
+  promptDallE();
 }
-
-async function promptDallE(input) {
-  document.getElementById("loader").style.display = "block";
-  document.getElementById("input").value = "";
-  document.getElementById("input").disabled = true;
 
+async function promptDallE() {
+  const loader = document.getElementById("loader");
+  const input = document.getElementById("input");
+  const generateBtn = document.getElementById("generate-btn");
+  const resultDiv = document.getElementById("result");
+
+  // Show loader and disable form
+  loader.style.display = "block";
+  input.disabled = true;
+  generateBtn.disabled = true;
+
+  // Store the prompt for later restoration
+  const prompt = input.value.trim();
+  if (!prompt) {
+    alert("Please enter a prompt");
+    loader.style.display = "none";
+    input.disabled = false;
+    generateBtn.disabled = false;
+    return;
+  }
+
+  // Collect all form values
   const model = document.getElementById("image-model").value;
   const size = document.getElementById("image-size").value;
-  const response = await fetch("v1/images/generations", {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-    },
-    body: JSON.stringify({
-      model: model,
-      steps: 10,
-      prompt: input,
-      n: 1,
-      size: size,
-    }),
-  });
-  const json = await response.json();
-  if (json.error) {
-    // Display error if there is one
-    var div = document.getElementById('result');  // Get the div by its ID
-    div.innerHTML = '<p style="color:red;">' + json.error.message + '</p>';
+  const negativePrompt = document.getElementById("negative-prompt").value.trim();
+  const n = parseInt(document.getElementById("image-count").value) || 1;
+  const stepInput = document.getElementById("image-steps").value.trim();
+  const step = stepInput ? parseInt(stepInput) : undefined;
+  const seedInput = document.getElementById("image-seed").value.trim();
+  const seed = seedInput ? parseInt(seedInput) : undefined;
+
+  // Prepare request body
+  // Combine prompt and negative prompt with "|" separator (backend expects this format)
+  let combinedPrompt = prompt;
+  if (negativePrompt) {
+    combinedPrompt = prompt + "|" + negativePrompt;
+  }
+
+  const requestBody = {
+    model: model,
+    prompt: combinedPrompt,
+    n: n,
+    size: size,
+  };
+
+  if (step !== undefined) {
+    requestBody.step = step;
+  }
+
+  if (seed !== undefined) {
+    requestBody.seed = seed;
+  }
+
+  // Handle file inputs
+  try {
+    // Source image (single file for img2img)
+    const sourceImageInput = document.getElementById("source-image");
+    if (sourceImageInput.files.length > 0) {
+      const base64 = await fileToBase64(sourceImageInput.files[0]);
+      requestBody.file = base64;
+    }
+
+    // Reference images (collect from all dynamic inputs)
+    const refImageInputs = document.querySelectorAll('.reference-image-file');
+    const refImageFiles = [];
+    for (const input of refImageInputs) {
+      if (input.files.length > 0) {
+        refImageFiles.push(input.files[0]);
+      }
+    }
+    if (refImageFiles.length > 0) {
+      const base64Array = await filesToBase64Array(refImageFiles);
+      requestBody.ref_images = base64Array;
+    }
+  } catch (error) {
+    console.error("Error processing image files:", error);
+    resultDiv.innerHTML = '<p class="text-red-500">Error processing image files: ' + error.message + '</p>';
+    loader.style.display = "none";
+    input.disabled = false;
+    generateBtn.disabled = false;
     return;
   }
-  const url = json.data[0].url;
 
-  var div = document.getElementById('result');  // Get the div by its ID
-  var img = document.createElement('img');         // Create a new img element
-  img.src = url;  // Set the source of the image
-  img.alt = 'Generated image';            // Set the alt text of the image
+  // Make API request
+  try {
+    const response = await fetch("v1/images/generations", {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify(requestBody),
+    });
 
-  div.innerHTML = '';                             // Clear the existing content of the div
-  div.appendChild(img);                           // Add the new img element to the div
+    const json = await response.json();
 
-  document.getElementById("loader").style.display = "none";
-  document.getElementById("input").disabled = false;
-  document.getElementById("input").focus();
+    if (json.error) {
+      // Display error
+      resultDiv.innerHTML = '<p class="text-red-500 p-4">Error: ' + json.error.message + '</p>';
+      loader.style.display = "none";
+      input.disabled = false;
+      generateBtn.disabled = false;
+      return;
+    }
+
+    // Clear result div
+    resultDiv.innerHTML = '';
+
+    // Display all generated images
+    if (json.data && json.data.length > 0) {
+      json.data.forEach((item, index) => {
+        const imageContainer = document.createElement("div");
+        imageContainer.className = "mb-6 bg-[var(--color-bg-primary)]/50 border border-[#1E293B] rounded-xl p-4";
+
+        // Create image element
+        const img = document.createElement("img");
+        if (item.url) {
+          img.src = item.url;
+        } else if (item.b64_json) {
+          img.src = "data:image/png;base64," + item.b64_json;
+        } else {
+          return; // Skip invalid items
+        }
+        img.alt = prompt;
+        img.className = "w-full h-auto rounded-lg mb-3";
+        imageContainer.appendChild(img);
+
+        // Create caption container
+        const captionDiv = document.createElement("div");
+        captionDiv.className = "mt-3 p-3 bg-[var(--color-bg-secondary)] rounded-lg";
+
+        // Prompt caption
+        const promptCaption = document.createElement("p");
+        promptCaption.className = "text-sm text-[var(--color-text-primary)] mb-2";
+        promptCaption.innerHTML = '<strong>Prompt:</strong> ' + escapeHtml(prompt);
+        captionDiv.appendChild(promptCaption);
+
+        // Negative prompt if provided
+        if (negativePrompt) {
+          const negativeCaption = document.createElement("p");
+          negativeCaption.className = "text-sm text-[var(--color-text-secondary)] mb-2";
+          negativeCaption.innerHTML = '<strong>Negative Prompt:</strong> ' + escapeHtml(negativePrompt);
+          captionDiv.appendChild(negativeCaption);
+        }
+
+        // Generation details
+        const detailsDiv = document.createElement("div");
+        detailsDiv.className = "flex flex-wrap gap-4 text-xs text-[var(--color-text-secondary)] mt-2";
+        detailsDiv.innerHTML = `
+          <span><strong>Size:</strong> ${size}</span>
+          ${step !== undefined ? `<span><strong>Steps:</strong> ${step}</span>` : ''}
+          ${seed !== undefined ? `<span><strong>Seed:</strong> ${seed}</span>` : ''}
+        `;
+        captionDiv.appendChild(detailsDiv);
+
+        // Copy prompt button
+        const copyBtn = document.createElement("button");
+        copyBtn.className = "mt-2 px-3 py-1 text-xs bg-[var(--color-primary)] text-white rounded hover:opacity-80";
+        copyBtn.innerHTML = '<i class="fas fa-copy mr-1"></i>Copy Prompt';
+        copyBtn.onclick = () => {
+          navigator.clipboard.writeText(prompt).then(() => {
+            copyBtn.innerHTML = '<i class="fas fa-check mr-1"></i>Copied!';
+            setTimeout(() => {
+              copyBtn.innerHTML = '<i class="fas fa-copy mr-1"></i>Copy Prompt';
+            }, 2000);
+          });
+        };
+        captionDiv.appendChild(copyBtn);
+
+        imageContainer.appendChild(captionDiv);
+        resultDiv.appendChild(imageContainer);
+      });
+    } else {
+      resultDiv.innerHTML = '<p class="text-[var(--color-text-secondary)] p-4">No images were generated.</p>';
+    }
+
+    // Preserve prompt in input field (don't clear it)
+    // The prompt is already in the input field, so we don't need to restore it
+
+  } catch (error) {
+    console.error("Error generating image:", error);
+    resultDiv.innerHTML = '<p class="text-red-500 p-4">Error: ' + error.message + '</p>';
+  } finally {
+    // Hide loader and re-enable form
+    loader.style.display = "none";
+    input.disabled = false;
+    generateBtn.disabled = false;
+    input.focus();
+  }
+}
+
+// Helper function to escape HTML
+function escapeHtml(text) {
+  const div = document.createElement("div");
+  div.textContent = text;
+  return div.innerHTML;
 }
 
-document.getElementById("input").focus();
-document.getElementById("genimage").addEventListener("submit", genImage);
+// Initialize
+document.addEventListener("DOMContentLoaded", function() {
+  const input = document.getElementById("input");
+  const form = document.getElementById("genimage");
+
+  if (input) {
+    input.focus();
+  }
+
+  if (form) {
+    form.addEventListener("submit", genImage);
+  }
 
-// Handle Enter key press in the prompt input
-document.getElementById("input").addEventListener("keypress", function(event) {
-    if (event.key === "Enter") {
+  // Handle Enter key press in the prompt input (but allow Shift+Enter for new lines)
+  if (input) {
+    input.addEventListener("keydown", function(event) {
+      if (event.key === "Enter" && !event.shiftKey) {
         event.preventDefault();
         genImage(event);
-    }
-});
+      }
+    });
+  }
 
-document.getElementById("loader").style.display = "none";
+  // Hide loader initially
+  const loader = document.getElementById("loader");
+  if (loader) {
+    loader.style.display = "none";
+  }
+});