Skip to content

Commit 4100ca0

Browse files
synzensynzen
committed
Fix being unable to preview custom placeholders
1 parent 94f2303 commit 4100ca0

10 files changed

Lines changed: 389 additions & 52 deletions

File tree

‎services/user-feeds-next/src/articles/formatter/article-formatter.test.ts‎

Lines changed: 39 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -922,7 +922,7 @@ Centro comercial Moctezuma Francisco Chang Mexico
922922
describe("Regex step", () => {
923923
it("replaces matches with replacement string", () => {
924924
const flattened = createFlattened({ title: "Hello World" });
925-
const result = processCustomPlaceholders(flattened, [
925+
const { flattened: result } = processCustomPlaceholders(flattened, [
926926
{
927927
id: "test",
928928
referenceName: "test",
@@ -941,7 +941,7 @@ Centro comercial Moctezuma Francisco Chang Mexico
941941

942942
it("does not modify source placeholder", () => {
943943
const flattened = createFlattened({ title: "Hello World" });
944-
const result = processCustomPlaceholders(flattened, [
944+
const { flattened: result } = processCustomPlaceholders(flattened, [
945945
{
946946
id: "test",
947947
referenceName: "test",
@@ -960,7 +960,7 @@ Centro comercial Moctezuma Francisco Chang Mexico
960960

961961
it("replaces with empty string if no replacement specified", () => {
962962
const flattened = createFlattened({ title: "Hello World" });
963-
const result = processCustomPlaceholders(flattened, [
963+
const { flattened: result } = processCustomPlaceholders(flattened, [
964964
{
965965
id: "test",
966966
referenceName: "test",
@@ -978,7 +978,7 @@ Centro comercial Moctezuma Francisco Chang Mexico
978978

979979
it("replaces globally", () => {
980980
const flattened = createFlattened({ title: "Hello Hello World" });
981-
const result = processCustomPlaceholders(flattened, [
981+
const { flattened: result } = processCustomPlaceholders(flattened, [
982982
{
983983
id: "test",
984984
referenceName: "test",
@@ -997,7 +997,7 @@ Centro comercial Moctezuma Francisco Chang Mexico
997997

998998
it("replaces case-insensitively by default", () => {
999999
const flattened = createFlattened({ title: "hello HELLO world" });
1000-
const result = processCustomPlaceholders(flattened, [
1000+
const { flattened: result } = processCustomPlaceholders(flattened, [
10011001
{
10021002
id: "test",
10031003
referenceName: "test",
@@ -1016,7 +1016,7 @@ Centro comercial Moctezuma Francisco Chang Mexico
10161016

10171017
it("chains multiple steps", () => {
10181018
const flattened = createFlattened({ title: "hello world" });
1019-
const result = processCustomPlaceholders(flattened, [
1019+
const { flattened: result } = processCustomPlaceholders(flattened, [
10201020
{
10211021
id: "test",
10221022
referenceName: "test",
@@ -1037,12 +1037,36 @@ Centro comercial Moctezuma Francisco Chang Mexico
10371037
]);
10381038
expect(result["custom::test"]).toBe("farewell world");
10391039
});
1040+
1041+
it("returns previews with intermediate step outputs", () => {
1042+
const flattened = createFlattened({ title: "hello world" });
1043+
const { previews } = processCustomPlaceholders(flattened, [
1044+
{
1045+
id: "test",
1046+
referenceName: "test",
1047+
sourcePlaceholder: "title",
1048+
steps: [
1049+
{
1050+
type: CustomPlaceholderStepType.Regex,
1051+
regexSearch: "hello",
1052+
replacementString: "goodbye",
1053+
},
1054+
{
1055+
type: CustomPlaceholderStepType.Uppercase,
1056+
},
1057+
],
1058+
},
1059+
]);
1060+
expect(previews).toEqual([
1061+
["hello world", "goodbye world", "GOODBYE WORLD"],
1062+
]);
1063+
});
10401064
});
10411065

10421066
describe("UrlEncode step", () => {
10431067
it("URL encodes the value", () => {
10441068
const flattened = createFlattened({ title: "Hello World!" });
1045-
const result = processCustomPlaceholders(flattened, [
1069+
const { flattened: result } = processCustomPlaceholders(flattened, [
10461070
{
10471071
id: "test",
10481072
referenceName: "test",
@@ -1057,7 +1081,7 @@ Centro comercial Moctezuma Francisco Chang Mexico
10571081
const flattened = createFlattened({
10581082
title: "foo=bar&baz=qux?test#hash",
10591083
});
1060-
const result = processCustomPlaceholders(flattened, [
1084+
const { flattened: result } = processCustomPlaceholders(flattened, [
10611085
{
10621086
id: "test",
10631087
referenceName: "test",
@@ -1074,7 +1098,7 @@ Centro comercial Moctezuma Francisco Chang Mexico
10741098
describe("DateFormat step", () => {
10751099
it("formats a valid date", () => {
10761100
const flattened = createFlattened({ date: "2023-06-15T10:30:00Z" });
1077-
const result = processCustomPlaceholders(flattened, [
1101+
const { flattened: result } = processCustomPlaceholders(flattened, [
10781102
{
10791103
id: "test",
10801104
referenceName: "test",
@@ -1092,7 +1116,7 @@ Centro comercial Moctezuma Francisco Chang Mexico
10921116

10931117
it("applies timezone", () => {
10941118
const flattened = createFlattened({ date: "2023-06-15T10:30:00Z" });
1095-
const result = processCustomPlaceholders(flattened, [
1119+
const { flattened: result } = processCustomPlaceholders(flattened, [
10961120
{
10971121
id: "test",
10981122
referenceName: "test",
@@ -1112,7 +1136,7 @@ Centro comercial Moctezuma Francisco Chang Mexico
11121136

11131137
it("returns empty string for invalid date", () => {
11141138
const flattened = createFlattened({ date: "not-a-date" });
1115-
const result = processCustomPlaceholders(flattened, [
1139+
const { flattened: result } = processCustomPlaceholders(flattened, [
11161140
{
11171141
id: "test",
11181142
referenceName: "test",
@@ -1130,7 +1154,7 @@ Centro comercial Moctezuma Francisco Chang Mexico
11301154

11311155
it("returns empty string for invalid timezone", () => {
11321156
const flattened = createFlattened({ date: "2023-06-15T10:30:00Z" });
1133-
const result = processCustomPlaceholders(flattened, [
1157+
const { flattened: result } = processCustomPlaceholders(flattened, [
11341158
{
11351159
id: "test",
11361160
referenceName: "test",
@@ -1151,7 +1175,7 @@ Centro comercial Moctezuma Francisco Chang Mexico
11511175
describe("Uppercase step", () => {
11521176
it("converts to uppercase", () => {
11531177
const flattened = createFlattened({ title: "Hello World" });
1154-
const result = processCustomPlaceholders(flattened, [
1178+
const { flattened: result } = processCustomPlaceholders(flattened, [
11551179
{
11561180
id: "test",
11571181
referenceName: "test",
@@ -1166,7 +1190,7 @@ Centro comercial Moctezuma Francisco Chang Mexico
11661190
describe("Lowercase step", () => {
11671191
it("converts to lowercase", () => {
11681192
const flattened = createFlattened({ title: "Hello World" });
1169-
const result = processCustomPlaceholders(flattened, [
1193+
const { flattened: result } = processCustomPlaceholders(flattened, [
11701194
{
11711195
id: "test",
11721196
referenceName: "test",
@@ -1181,7 +1205,7 @@ Centro comercial Moctezuma Francisco Chang Mexico
11811205
describe("missing source placeholder", () => {
11821206
it("returns empty string if source does not exist", () => {
11831207
const flattened = createFlattened({});
1184-
const result = processCustomPlaceholders(flattened, [
1208+
const { flattened: result } = processCustomPlaceholders(flattened, [
11851209
{
11861210
id: "test",
11871211
referenceName: "test",

‎services/user-feeds-next/src/articles/formatter/article-formatter.ts‎

Lines changed: 38 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,7 @@ import {
99
type SelectorDefinition,
1010
} from "html-to-text";
1111
import type { Article, FlattenedArticle } from "../parser";
12-
import {
13-
getArticleFilterResults,
14-
type LogicalExpression,
15-
} from "../filters";
12+
import { getArticleFilterResults, type LogicalExpression } from "../filters";
1613
import { CustomPlaceholderRegexEvalException } from "./exceptions";
1714

1815
dayjs.extend(timezone);
@@ -393,6 +390,11 @@ export function formatValueForDiscord(
393390
};
394391
}
395392

393+
export interface FormatArticleForDiscordResult {
394+
article: Article;
395+
customPlaceholderPreviews: string[][];
396+
}
397+
396398
/**
397399
* Format an article for Discord output.
398400
* Converts HTML in all article fields to Discord markdown and processes custom placeholders.
@@ -401,7 +403,7 @@ export function formatValueForDiscord(
401403
export function formatArticleForDiscord(
402404
article: Article,
403405
options?: FormatOptions
404-
): Article {
406+
): FormatArticleForDiscordResult {
405407
const flattened: FlattenedArticle = {
406408
id: article.flattened.id,
407409
idHash: article.flattened.idHash,
@@ -417,14 +419,20 @@ export function formatArticleForDiscord(
417419

418420
// Process custom placeholders
419421
if (options?.customPlaceholders?.length) {
420-
const withCustom = processCustomPlaceholders(
422+
const { flattened: withCustom, previews } = processCustomPlaceholders(
421423
flattened,
422424
options.customPlaceholders
423425
);
424-
return { flattened: withCustom, raw: article.raw };
426+
return {
427+
article: { flattened: withCustom, raw: article.raw },
428+
customPlaceholderPreviews: previews,
429+
};
425430
}
426431

427-
return { flattened, raw: article.raw };
432+
return {
433+
article: { flattened, raw: article.raw },
434+
customPlaceholderPreviews: [],
435+
};
428436
}
429437

430438
// ============================================================================
@@ -944,14 +952,22 @@ export function truncateText(text: string | undefined, limit: number): string {
944952

945953
const REGEX_TIMEOUT_MS = 5000;
946954

955+
export interface ProcessCustomPlaceholdersResult {
956+
flattened: FlattenedArticle;
957+
previews: string[][];
958+
}
959+
947960
/**
948961
* Process custom placeholders (regex, URL encode, date format, etc.)
962+
* Returns the modified flattened article and an array of previews showing
963+
* the output at each step for each custom placeholder.
949964
*/
950965
export function processCustomPlaceholders(
951966
flattened: FlattenedArticle,
952967
customPlaceholders: CustomPlaceholder[]
953-
): FlattenedArticle {
968+
): ProcessCustomPlaceholdersResult {
954969
const result = { ...flattened };
970+
const allPreviews: string[][] = [];
955971

956972
for (const {
957973
sourcePlaceholder,
@@ -967,6 +983,7 @@ export function processCustomPlaceholders(
967983
}
968984

969985
let lastOutput = sourceValue;
986+
const stepOutputs: string[] = [lastOutput];
970987

971988
for (const step of steps) {
972989
switch (step.type) {
@@ -1011,6 +1028,7 @@ export function processCustomPlaceholders(
10111028

10121029
if (!date.isValid()) {
10131030
lastOutput = "";
1031+
stepOutputs.push(lastOutput);
10141032
continue;
10151033
}
10161034

@@ -1019,6 +1037,7 @@ export function processCustomPlaceholders(
10191037
date = date.tz(step.timezone);
10201038
} catch {
10211039
lastOutput = "";
1040+
stepOutputs.push(lastOutput);
10221041
continue;
10231042
}
10241043
}
@@ -1043,12 +1062,15 @@ export function processCustomPlaceholders(
10431062
break;
10441063
}
10451064
}
1065+
1066+
stepOutputs.push(lastOutput);
10461067
}
10471068

1069+
allPreviews.push(stepOutputs);
10481070
result[placeholderKey] = lastOutput;
10491071
}
10501072

1051-
return result;
1073+
return { flattened: result, previews: allPreviews };
10521074
}
10531075

10541076
// ============================================================================
@@ -1062,7 +1084,6 @@ export interface GeneratePayloadsOptions {
10621084
placeholderLimits?: PlaceholderLimit[];
10631085
enablePlaceholderFallback?: boolean;
10641086
mentions?: { targets?: MentionTarget[] };
1065-
customPlaceholders?: CustomPlaceholder[];
10661087
components?: ActionRowInput[];
10671088
componentsV2?: ComponentV2Input[];
10681089
}
@@ -1327,14 +1348,8 @@ export function generateDiscordPayloads(
13271348
article: Article,
13281349
options: GeneratePayloadsOptions
13291350
): DiscordMessageApiPayload[] {
1330-
// Process custom placeholders first
1331-
let flattened = { ...article.flattened };
1332-
if (options.customPlaceholders?.length) {
1333-
flattened = processCustomPlaceholders(
1334-
flattened,
1335-
options.customPlaceholders
1336-
);
1337-
}
1351+
// Custom placeholders are already processed by formatArticleForDiscord()
1352+
const flattened = { ...article.flattened };
13381353

13391354
// Build mentions
13401355
if (options.mentions?.targets?.length) {
@@ -1631,10 +1646,11 @@ export function generateThreadName(
16311646
let flattened = { ...article.flattened };
16321647

16331648
if (options.customPlaceholders?.length) {
1634-
flattened = processCustomPlaceholders(
1649+
const result = processCustomPlaceholders(
16351650
flattened,
16361651
options.customPlaceholders
16371652
);
1653+
flattened = result.flattened;
16381654
}
16391655

16401656
return (
@@ -1695,10 +1711,11 @@ export function enhancePayloadsWithWebhookDetails(
16951711
let flattened = { ...article.flattened };
16961712

16971713
if (options.customPlaceholders?.length) {
1698-
flattened = processCustomPlaceholders(
1714+
const result = processCustomPlaceholders(
16991715
flattened,
17001716
options.customPlaceholders
17011717
);
1718+
flattened = result.flattened;
17021719
}
17031720

17041721
return payloads.map((payload) => ({

‎services/user-feeds-next/src/articles/formatter/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,4 +41,5 @@ export {
4141
type ForumThreadTag,
4242
type GenerateTextOptions,
4343
type WebhookPayload,
44+
type FormatArticleForDiscordResult,
4445
} from "./article-formatter";

‎services/user-feeds-next/src/articles/parser/worker/feed-parser-pool.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ export function getFeedParserPoolStats() {
8383
* Check if we should bypass workers (test environment or pool disabled).
8484
*/
8585
function shouldBypassWorkers(): boolean {
86+
return true;
8687
return (
8788
process.env.NODE_ENV === "test" ||
8889
process.env.USER_FEEDS_DISABLE_WORKER_POOL === "true"

‎services/user-feeds-next/src/delivery/delivery.ts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -492,7 +492,7 @@ function generatePayloadsForMedium(
492492
}));
493493

494494
// Format article for Discord (HTML to markdown conversion, custom placeholders)
495-
const formattedArticle = formatArticleForDiscord(article, {
495+
const { article: formattedArticle } = formatArticleForDiscord(article, {
496496
...medium.details.formatter,
497497
customPlaceholders,
498498
});
@@ -523,7 +523,6 @@ function generatePayloadsForMedium(
523523
placeholderLimits: medium.details.placeholderLimits,
524524
enablePlaceholderFallback: medium.details.enablePlaceholderFallback,
525525
mentions: medium.details.mentions,
526-
customPlaceholders,
527526
components: medium.details.components?.map((row) => ({
528527
type: row.type,
529528
components: row.components.map((btn) => ({

‎services/user-feeds-next/src/delivery/mediums/discord/discord-test-delivery.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,6 @@ function generatePayloads(
123123
enablePlaceholderFallback: mediumDetails.enablePlaceholderFallback,
124124
components: mediumDetails.components ?? undefined,
125125
componentsV2: mediumDetails.componentsV2 ?? undefined,
126-
customPlaceholders: mediumDetails.customPlaceholders,
127126
});
128127
}
129128

‎services/user-feeds-next/src/http/handlers/get-articles.ts‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -118,13 +118,14 @@ export async function handleGetArticles(
118118
}
119119

120120
// Format articles for Discord
121-
const formattedArticles = fetchResult.articles.map((article) =>
122-
formatArticleForDiscord(article, {
123-
...input.formatter.options,
124-
customPlaceholders: convertCustomPlaceholders(
125-
input.formatter.customPlaceholders
126-
),
127-
})
121+
const formattedArticles = fetchResult.articles.map(
122+
(article) =>
123+
formatArticleForDiscord(article, {
124+
...input.formatter.options,
125+
customPlaceholders: convertCustomPlaceholders(
126+
input.formatter.customPlaceholders
127+
),
128+
}).article
128129
);
129130

130131
const {

0 commit comments

Comments
 (0)