Improve image detection for templates, fix strip images not applying

Author: synzen

services/backend-api/client/src/features/feedConnections/api/createDiscordChannelConnection.ts

@@ -70,6 +70,12 @@ export interface CreateDiscordChannelConnectionInput {
       characterCount: number;
       appendString?: string | null;
     }> | null;
+    formatter?: {
+      formatTables?: boolean | null;
+      stripImages?: boolean | null;
+      disableImageLinkPreviews?: boolean | null;
+      ignoreNewLines?: boolean | null;
+    } | null;
   };
 }
 

services/backend-api/client/src/features/feedConnections/components/AddConnectionDialog/DiscordApplicationWebhookConnectionDialogContent.tsx

@@ -143,6 +143,7 @@ export const DiscordApplicationWebhookConnectionDialogContent: React.FC<Props> =
         embeds: templateData.embeds,
         componentsV2: templateData.componentsV2,
         placeholderLimits: templateData.placeholderLimits,
+        formatter: templateData.formatter,
       },
     });
   }, [feedId, watch, mutateAsync, selectedTemplateId, channelId, detectedFields]);
@@ -214,6 +215,7 @@ export const DiscordApplicationWebhookConnectionDialogContent: React.FC<Props> =
           embeds: templateData.embeds,
           componentsV2: templateData.componentsV2,
           placeholderLimits: templateData.placeholderLimits,
+          formatter: templateData.formatter,
         },
       });
 

services/backend-api/client/src/features/feedConnections/components/AddConnectionDialog/DiscordForumChannelConnectionDialogContent.tsx

@@ -146,6 +146,7 @@ export const DiscordForumChannelConnectionDialogContent: React.FC<Props> = ({
         embeds: templateData.embeds,
         componentsV2: templateData.componentsV2,
         placeholderLimits: templateData.placeholderLimits,
+        formatter: templateData.formatter,
       },
     });
   }, [feedId, watch, mutateAsync, selectedTemplateId, detectedFields]);
@@ -247,6 +248,7 @@ export const DiscordForumChannelConnectionDialogContent: React.FC<Props> = ({
         embeds: templateData.embeds,
         componentsV2: templateData.componentsV2,
         placeholderLimits: templateData.placeholderLimits,
+        formatter: templateData.formatter,
       },
     });
 

services/backend-api/client/src/features/feedConnections/components/AddConnectionDialog/DiscordTextChannelConnectionDialogContent.tsx

@@ -175,6 +175,7 @@ export const DiscordTextChannelConnectionDialogContent: React.FC<Props> = ({
         embeds: templateData.embeds,
         componentsV2: templateData.componentsV2,
         placeholderLimits: templateData.placeholderLimits,
+        formatter: templateData.formatter,
       },
     });
   }, [feedId, watch, mutateAsync, selectedTemplateId, detectedFields]);
@@ -276,6 +277,7 @@ export const DiscordTextChannelConnectionDialogContent: React.FC<Props> = ({
           embeds: templateData.embeds,
           componentsV2: templateData.componentsV2,
           placeholderLimits: templateData.placeholderLimits,
+          formatter: templateData.formatter,
         },
       });
 

services/backend-api/client/src/features/templates/constants/templates.ts

@@ -105,18 +105,6 @@ export const RICH_EMBED_TEMPLATE: Template = {
               name: "Description",
               content: `{{${descriptionField}}}`,
             },
-            ...(linkField
-              ? [
-                  {
-                    type: ComponentType.V2Divider as const,
-                    id: "rich-embed-divider",
-                    name: "Divider",
-                    visual: true,
-                    spacing: 1 as const,
-                    children: [] as [],
-                  },
-                ]
-              : []),
             ...(linkField
               ? [
                   {

services/backend-api/client/src/features/templates/utils/detectImageField.test.ts

@@ -52,6 +52,27 @@ describe("detectImageFields", () => {
       ];
       expect(detectImageFields(articles)).toEqual(["image"]);
     });
+
+    it("skips values with whitespace (mixed content)", () => {
+      const articles = [
+        {
+          mixedContent:
+            "https://example.com/photo.png?query=1 submitted by /u/Someone [link] [comments]",
+          image: "https://example.com/actual.jpg",
+        },
+      ];
+      expect(detectImageFields(articles)).toEqual(["image"]);
+    });
+
+    it("skips values with newlines", () => {
+      const articles = [
+        {
+          multiline: "https://example.com/photo.jpg\nSome other text",
+          image: "https://example.com/actual.jpg",
+        },
+      ];
+      expect(detectImageFields(articles)).toEqual(["image"]);
+    });
   });
 
   describe("deduplication", () => {
@@ -152,6 +173,71 @@ describe("detectImageFields", () => {
     });
   });
 
+  describe("Reddit CDN deduplication", () => {
+    it("deduplicates preview.redd.it and i.redd.it URLs with same filename", () => {
+      const articles = [
+        {
+          image__url: "https://i.redd.it/7azztrbwusag1.png",
+          "extracted::description::anchor3":
+            "https://preview.redd.it/7azztrbwusag1.png?width=640&crop=smart&auto=webp&s=abc123",
+        },
+      ];
+      expect(detectImageFields(articles)).toEqual(["image__url"]);
+    });
+
+    it("deduplicates same Reddit image with different query parameters", () => {
+      const articles = [
+        {
+          thumbnail: "https://preview.redd.it/abc123.jpg?width=108&crop=smart",
+          image: "https://preview.redd.it/abc123.jpg?width=1080&format=png",
+        },
+      ];
+      expect(detectImageFields(articles)).toEqual(["image"]);
+    });
+
+    it("keeps different Reddit images as separate", () => {
+      const articles = [
+        {
+          image1: "https://i.redd.it/first.png",
+          image2: "https://i.redd.it/second.png",
+        },
+      ];
+      expect(detectImageFields(articles)).toEqual(["image1", "image2"]);
+    });
+
+    it("deduplicates redditmedia.com URLs by filename", () => {
+      const articles = [
+        {
+          thumb: "https://a.thumbs.redditmedia.com/abc123.jpg",
+          full: "https://i.redditmedia.com/abc123.jpg",
+        },
+      ];
+      expect(detectImageFields(articles)).toEqual(["full"]);
+    });
+  });
+
+  describe("query parameter normalization", () => {
+    it("deduplicates same image with different query parameters", () => {
+      const articles = [
+        {
+          thumb: "https://example.com/photo.jpg?size=small",
+          full: "https://example.com/photo.jpg?size=large",
+        },
+      ];
+      expect(detectImageFields(articles)).toEqual(["full"]);
+    });
+
+    it("deduplicates image with and without query parameters", () => {
+      const articles = [
+        {
+          clean: "https://example.com/photo.jpg",
+          withParams: "https://example.com/photo.jpg?v=123",
+        },
+      ];
+      expect(detectImageFields(articles)).toEqual(["clean"]);
+    });
+  });
+
   describe("supported image extensions", () => {
     const extensions = ["jpg", "jpeg", "png", "gif", "webp", "svg", "bmp", "ico"];
 

services/backend-api/client/src/features/templates/utils/detectImageField.ts

@@ -17,37 +17,70 @@ function selectPreferredField(fields: string[]): string {
 function isImageUrl(value: unknown): boolean {
   if (!value || typeof value !== "string") return false;
 
+  const trimmed = value.trim();
+
+  // Reject values with whitespace (indicates mixed content, not a pure URL)
+  if (/\s/.test(trimmed)) return false;
+
   try {
-    const url = new URL(value);
+    const url = new URL(trimmed);
 
     return IMAGE_EXTENSIONS.test(url.pathname);
   } catch {
     return false;
   }
 }
 
+/**
+ * Extracts a normalized key for deduplication purposes.
+ * This handles cases where the same image appears with different subdomains
+ * or query parameters (e.g., Reddit's preview.redd.it vs i.redd.it).
+ */
+function getImageDedupeKey(urlString: string): string {
+  try {
+    const url = new URL(urlString);
+
+    // Extract just the filename from the pathname
+    const pathParts = url.pathname.split("/");
+    const filename = pathParts[pathParts.length - 1];
+
+    // For Reddit CDN URLs, normalize by using just the filename
+    // since preview.redd.it and i.redd.it serve the same images
+    if (url.hostname.endsWith("redd.it") || url.hostname.endsWith("redditmedia.com")) {
+      return `reddit:${filename}`;
+    }
+
+    // For other URLs, use hostname + pathname (without query params)
+    // This handles cases where the same image has different query params
+    return `${url.hostname}${url.pathname}`;
+  } catch {
+    return urlString;
+  }
+}
+
 export function detectImageFields(articles: Array<Record<string, unknown>>): string[] {
   if (!articles || articles.length === 0) return [];
 
   const selectedFields = new Set<string>();
 
   for (const article of articles) {
-    // Group fields by their URL value within this article
-    const urlToFields = new Map<string, string[]>();
+    // Group fields by their normalized URL key within this article
+    const dedupeKeyToFields = new Map<string, string[]>();
 
     for (const [field, value] of Object.entries(article)) {
       if (field === "id" || field === "idHash") continue;
 
       if (isImageUrl(value)) {
         const url = value as string;
-        const existing = urlToFields.get(url) || [];
+        const dedupeKey = getImageDedupeKey(url);
+        const existing = dedupeKeyToFields.get(dedupeKey) || [];
         existing.push(field);
-        urlToFields.set(url, existing);
+        dedupeKeyToFields.set(dedupeKey, existing);
       }
     }
 
-    // For each unique URL, select the preferred field
-    for (const fields of urlToFields.values()) {
+    // For each unique image (by dedupe key), select the preferred field
+    for (const fields of dedupeKeyToFields.values()) {
       const preferredField = selectPreferredField(fields);
       selectedFields.add(preferredField);
     }

services/backend-api/src/features/feed-connections/dto/create-discord-channel-connection-input.dto.ts

@@ -10,7 +10,11 @@ import {
   ValidateIf,
   ValidateNested,
 } from "class-validator";
-import { DiscordEmbed, DiscordPlaceholderLimitOptions } from "../../../common";
+import {
+  DiscordConnectionFormatterOptions,
+  DiscordEmbed,
+  DiscordPlaceholderLimitOptions,
+} from "../../../common";
 
 class Webhook {
   @IsString()
@@ -94,4 +98,11 @@ export class CreateDiscordChnnnelConnectionInputDto {
   @ValidateNested({ each: true })
   @Type(() => DiscordPlaceholderLimitOptions)
   placeholderLimits?: DiscordPlaceholderLimitOptions[];
+
+  @IsOptional()
+  @Type(() => DiscordConnectionFormatterOptions)
+  @ValidateNested()
+  @IsObject()
+  @ValidateIf((v) => v !== null)
+  formatter?: DiscordConnectionFormatterOptions | null;
 }

services/backend-api/src/features/feed-connections/feed-connections-discord-channels.controller.ts

@@ -79,6 +79,7 @@ export class FeedConnectionsDiscordChannelsController {
       embeds,
       componentsV2,
       placeholderLimits,
+      formatter,
     }: CreateDiscordChnnnelConnectionInputDto,
     @DiscordAccessToken()
     { access_token, discord: { id: discordUserId } }: SessionAccessToken
@@ -98,6 +99,7 @@ export class FeedConnectionsDiscordChannelsController {
           embeds,
           componentsV2,
           placeholderLimits,
+          formatter: formatter || undefined,
         },
       }
     );

services/backend-api/src/features/feed-connections/feed-connections-discord-channels.service.ts

@@ -216,6 +216,7 @@ export class FeedConnectionsDiscordChannelsService {
       embeds?: DiscordChannelConnection["details"]["embeds"];
       componentsV2?: DiscordChannelConnection["details"]["componentsV2"];
       placeholderLimits?: DiscordChannelConnection["details"]["placeholderLimits"];
+      formatter?: DiscordChannelConnection["details"]["formatter"];
     };
   }): Promise<DiscordChannelConnection> {
     const connectionId = new Types.ObjectId();
@@ -362,6 +363,7 @@ export class FeedConnectionsDiscordChannelsService {
                 content: templateData?.content,
                 componentsV2: validatedComponentsV2,
                 placeholderLimits: templateData?.placeholderLimits,
+                formatter: templateData?.formatter || undefined,
               },
             },
           },