Merge branch 'main' into refactor/model-manager-2

Author: lstein

installer/lib/installer.py

@@ -460,10 +460,10 @@ def get_torch_source() -> (Union[str, None], str):
             url = "https://download.pytorch.org/whl/cpu"
 
     if device == "cuda":
-        url = "https://download.pytorch.org/whl/cu121"
+        url = "https://download.pytorch.org/whl/cu118"
         optional_modules = "[xformers,onnx-cuda]"
     if device == "cuda_and_dml":
-        url = "https://download.pytorch.org/whl/cu121"
+        url = "https://download.pytorch.org/whl/cu118"
         optional_modules = "[xformers,onnx-directml]"
 
     # in all other cases, Torch wheels should be coming from PyPi as of Torch 1.13

invokeai/app/invocations/metadata.py

@@ -160,13 +160,14 @@ class CoreMetadataInvocation(BaseInvocation):
     )
 
     # High resolution fix metadata.
-    hrf_width: Optional[int] = InputField(
+    hrf_enabled: Optional[float] = InputField(
         default=None,
-        description="The high resolution fix height and width multipler.",
+        description="Whether or not high resolution fix was enabled.",
     )
-    hrf_height: Optional[int] = InputField(
+    # TODO: should this be stricter or do we just let the UI handle it?
+    hrf_method: Optional[str] = InputField(
         default=None,
-        description="The high resolution fix height and width multipler.",
+        description="The high resolution fix upscale method.",
     )
     hrf_strength: Optional[float] = InputField(
         default=None,

invokeai/backend/install/model_install_backend.py

@@ -254,7 +254,13 @@ def heuristic_import(
         elif path.is_dir() and any(
             [
                 (path / x).exists()
-                for x in {"config.json", "model_index.json", "learned_embeds.bin", "pytorch_lora_weights.bin"}
+                for x in {
+                    "config.json",
+                    "model_index.json",
+                    "learned_embeds.bin",
+                    "pytorch_lora_weights.bin",
+                    "pytorch_lora_weights.safetensors",
+                }
             ]
         ):
             models_installed.update({str(model_path_id_or_url): self._install_path(path)})
@@ -357,7 +363,7 @@ def _install_repo(self, repo_id: str) -> AddModelResult:
                 for suffix in ["safetensors", "bin"]:
                     if f"{prefix}pytorch_lora_weights.{suffix}" in files:
                         location = self._download_hf_model(
-                            repo_id, ["pytorch_lora_weights.bin"], staging, subfolder=subfolder
+                            repo_id, [f"pytorch_lora_weights.{suffix}"], staging, subfolder=subfolder
                         )  # LoRA
                         break
                     elif (

invokeai/backend/model_management/lora.py

@@ -166,6 +166,15 @@ def apply_ti(
         init_tokens_count = None
         new_tokens_added = None
 
+        # TODO: This is required since Transformers 4.32 see
+        # https://github.com/huggingface/transformers/pull/25088
+        # More information by NVIDIA:
+        # https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html#requirements-tc
+        # This value might need to be changed in the future and take the GPUs model into account as there seem
+        # to be ideal values for different GPUS. This value is temporary!
+        # For references to the current discussion please see https://github.com/invoke-ai/InvokeAI/pull/4817
+        pad_to_multiple_of = 8
+
         try:
             # HACK: The CLIPTokenizer API does not include a way to remove tokens after calling add_tokens(...). As a
             # workaround, we create a full copy of `tokenizer` so that its original behavior can be restored after
@@ -175,7 +184,7 @@ def apply_ti(
             # but a pickle roundtrip was found to be much faster (1 sec vs. 0.05 secs).
             ti_tokenizer = pickle.loads(pickle.dumps(tokenizer))
             ti_manager = TextualInversionManager(ti_tokenizer)
-            init_tokens_count = text_encoder.resize_token_embeddings(None).num_embeddings
+            init_tokens_count = text_encoder.resize_token_embeddings(None, pad_to_multiple_of).num_embeddings
 
             def _get_trigger(ti_name, index):
                 trigger = ti_name
@@ -190,7 +199,7 @@ def _get_trigger(ti_name, index):
                     new_tokens_added += ti_tokenizer.add_tokens(_get_trigger(ti_name, i))
 
             # modify text_encoder
-            text_encoder.resize_token_embeddings(init_tokens_count + new_tokens_added)
+            text_encoder.resize_token_embeddings(init_tokens_count + new_tokens_added, pad_to_multiple_of)
             model_embeddings = text_encoder.get_input_embeddings()
 
             for ti_name, ti in ti_list:
@@ -222,7 +231,7 @@ def _get_trigger(ti_name, index):
 
         finally:
             if init_tokens_count and new_tokens_added:
-                text_encoder.resize_token_embeddings(init_tokens_count)
+                text_encoder.resize_token_embeddings(init_tokens_count, pad_to_multiple_of)
 
     @classmethod
     @contextmanager

invokeai/backend/model_management/model_probe.py

@@ -183,12 +183,13 @@ def get_model_type_from_folder(cls, folder_path: Path, model: ModelMixin) -> Mod
         if model:
             class_name = model.__class__.__name__
         else:
+            for suffix in ["bin", "safetensors"]:
+                if (folder_path / f"learned_embeds.{suffix}").exists():
+                    return ModelType.TextualInversion
+                if (folder_path / f"pytorch_lora_weights.{suffix}").exists():
+                    return ModelType.Lora
             if (folder_path / "unet/model.onnx").exists():
                 return ModelType.ONNX
-            if (folder_path / "learned_embeds.bin").exists():
-                return ModelType.TextualInversion
-            if (folder_path / "pytorch_lora_weights.bin").exists():
-                return ModelType.Lora
             if (folder_path / "image_encoder.txt").exists():
                 return ModelType.IPAdapter
 

invokeai/backend/model_management/models/lora.py

@@ -68,8 +68,9 @@ def detect_format(cls, path: str):
             raise ModelNotFoundException()
 
         if os.path.isdir(path):
-            if os.path.exists(os.path.join(path, "pytorch_lora_weights.bin")):
-                return LoRAModelFormat.Diffusers
+            for ext in ["safetensors", "bin"]:
+                if os.path.exists(os.path.join(path, f"pytorch_lora_weights.{ext}")):
+                    return LoRAModelFormat.Diffusers
 
         if os.path.isfile(path):
             if any([path.endswith(f".{ext}") for ext in ["safetensors", "ckpt", "pt"]]):
@@ -86,8 +87,10 @@ def convert_if_required(
         base_model: BaseModelType,
     ) -> str:
         if cls.detect_format(model_path) == LoRAModelFormat.Diffusers:
-            # TODO: add diffusers lora when it stabilizes a bit
-            raise NotImplementedError("Diffusers lora not supported")
+            for ext in ["safetensors", "bin"]:  # return path to the safetensors file inside the folder
+                path = Path(model_path, f"pytorch_lora_weights.{ext}")
+                if path.exists():
+                    return path
         else:
             return model_path
 

invokeai/frontend/web/public/locales/en.json

@@ -221,6 +221,19 @@
         "resetIPAdapterImage": "Reset IP Adapter Image",
         "ipAdapterImageFallback": "No IP Adapter Image Selected"
     },
+    "hrf": {
+        "hrf": "High Resolution Fix",
+        "enableHrf": "Enable High Resolution Fix",
+        "enableHrfTooltip": "Generate with a lower initial resolution, upscale to the base resolution, then run Image-to-Image.",
+        "upscaleMethod": "Upscale Method",
+        "hrfStrength": "High Resolution Fix Strength",
+        "strengthTooltip": "Lower values result in fewer details, which may reduce potential artifacts.",
+        "metadata": {
+            "enabled": "High Resolution Fix Enabled",
+            "strength": "High Resolution Fix Strength",
+            "method": "High Resolution Fix Method"
+        }
+    },
     "embedding": {
         "addEmbedding": "Add Embedding",
         "incompatibleModel": "Incompatible base model:",
@@ -1258,15 +1271,11 @@
         },
         "compositingBlur": {
             "heading": "Blur",
-            "paragraphs": [
-                "The blur radius of the mask."
-            ]
+            "paragraphs": ["The blur radius of the mask."]
         },
         "compositingBlurMethod": {
             "heading": "Blur Method",
-            "paragraphs": [
-                "The method of blur applied to the masked area."
-            ]
+            "paragraphs": ["The method of blur applied to the masked area."]
         },
         "compositingCoherencePass": {
             "heading": "Coherence Pass",
@@ -1276,9 +1285,7 @@
         },
         "compositingCoherenceMode": {
             "heading": "Mode",
-            "paragraphs": [
-                "The mode of the Coherence Pass."
-            ]
+            "paragraphs": ["The mode of the Coherence Pass."]
         },
         "compositingCoherenceSteps": {
             "heading": "Steps",
@@ -1296,9 +1303,7 @@
         },
         "compositingMaskAdjustments": {
             "heading": "Mask Adjustments",
-            "paragraphs": [
-                "Adjust the mask."
-            ]
+            "paragraphs": ["Adjust the mask."]
         },
         "controlNetBeginEnd": {
             "heading": "Begin / End Step Percentage",
@@ -1356,9 +1361,7 @@
         },
         "infillMethod": {
             "heading": "Infill Method",
-            "paragraphs": [
-                "Method to infill the selected area."
-            ]
+            "paragraphs": ["Method to infill the selected area."]
         },
         "lora": {
             "heading": "LoRA Weight",

invokeai/frontend/web/src/features/gallery/components/ImageMetadataViewer/ImageMetadataActions.tsx

@@ -35,6 +35,9 @@ const ImageMetadataActions = (props: Props) => {
     recallWidth,
     recallHeight,
     recallStrength,
+    recallHrfEnabled,
+    recallHrfStrength,
+    recallHrfMethod,
     recallLoRA,
     recallControlNet,
     recallIPAdapter,
@@ -81,6 +84,18 @@ const ImageMetadataActions = (props: Props) => {
     recallStrength(metadata?.strength);
   }, [metadata?.strength, recallStrength]);
 
+  const handleRecallHrfEnabled = useCallback(() => {
+    recallHrfEnabled(metadata?.hrf_enabled);
+  }, [metadata?.hrf_enabled, recallHrfEnabled]);
+
+  const handleRecallHrfStrength = useCallback(() => {
+    recallHrfStrength(metadata?.hrf_strength);
+  }, [metadata?.hrf_strength, recallHrfStrength]);
+
+  const handleRecallHrfMethod = useCallback(() => {
+    recallHrfMethod(metadata?.hrf_method);
+  }, [metadata?.hrf_method, recallHrfMethod]);
+
   const handleRecallLoRA = useCallback(
     (lora: LoRAMetadataItem) => {
       recallLoRA(lora);
@@ -225,6 +240,27 @@ const ImageMetadataActions = (props: Props) => {
           onClick={handleRecallStrength}
         />
       )}
+      {metadata.hrf_enabled && (
+        <ImageMetadataItem
+          label={t('hrf.metadata.enabled')}
+          value={metadata.hrf_enabled}
+          onClick={handleRecallHrfEnabled}
+        />
+      )}
+      {metadata.hrf_enabled && metadata.hrf_strength && (
+        <ImageMetadataItem
+          label={t('hrf.metadata.strength')}
+          value={metadata.hrf_strength}
+          onClick={handleRecallHrfStrength}
+        />
+      )}
+      {metadata.hrf_enabled && metadata.hrf_method && (
+        <ImageMetadataItem
+          label={t('hrf.metadata.method')}
+          value={metadata.hrf_method}
+          onClick={handleRecallHrfMethod}
+        />
+      )}
       {metadata.loras &&
         metadata.loras.map((lora, index) => {
           if (isValidLoRAModel(lora.lora)) {

invokeai/frontend/web/src/features/nodes/types/types.ts

@@ -1424,6 +1424,9 @@ export const zCoreMetadata = z
     loras: z.array(zLoRAMetadataItem).nullish().catch(null),
     vae: zVaeModelField.nullish().catch(null),
     strength: z.number().nullish().catch(null),
+    hrf_enabled: z.boolean().nullish().catch(null),
+    hrf_strength: z.number().nullish().catch(null),
+    hrf_method: z.string().nullish().catch(null),
     init_image: z.string().nullish().catch(null),
     positive_style_prompt: z.string().nullish().catch(null),
     negative_style_prompt: z.string().nullish().catch(null),