Use separate endpoint for delivery previews

Author: synzen

services/feed-requests/src/feed-fetcher/dto/fetch-feed-delivery-preview.dto.ts

@@ -0,0 +1,17 @@
+import { Type } from 'class-transformer';
+import { IsNumber, IsOptional, IsString, Validate } from 'class-validator';
+import { HttpValidator } from './fetch-feed.dto';
+
+export class FetchFeedDeliveryPreviewDto {
+  @Validate(HttpValidator)
+  url!: string;
+
+  @IsString()
+  @IsOptional()
+  lookupKey?: string;
+
+  @IsNumber()
+  @IsOptional()
+  @Type(() => Number)
+  stalenessThresholdSeconds?: number;
+}

services/feed-requests/src/feed-fetcher/dto/index.ts

@@ -1,4 +1,5 @@
 export * from './fetch-feed.dto';
 export * from './fetch-feed-details.dto';
+export * from './fetch-feed-delivery-preview.dto';
 export * from './get-feed-requests-input.dto';
 export * from './get-feed-requests-output.dto';

services/feed-requests/src/feed-fetcher/feed-fetcher.controller.ts

@@ -15,6 +15,7 @@ import { RequestStatus } from './constants';
 import {
   FetchFeedDto,
   FetchFeedDetailsDto,
+  FetchFeedDeliveryPreviewDto,
   GetFeedRequestsInputDto,
   GetFeedRequestsOutputDto,
 } from './dto';
@@ -103,6 +104,117 @@ export class FeedFetcherController {
     return this.getLatestRequest(data);
   }
 
+  @Post('feed-requests/delivery-preview')
+  @UseGuards(ApiGuard)
+  async fetchFeedDeliveryPreview(
+    @Body(ValidationPipe) data: FetchFeedDeliveryPreviewDto,
+  ): Promise<FetchFeedDetailsDto> {
+    const lookupKey = data.lookupKey || data.url;
+
+    // 1. Get latest request (any status, including errors)
+    let latestRequest =
+      await this.partitionedRequestsStoreService.getLatestRequestAnyStatus(
+        lookupKey,
+      );
+
+    // 2. Check staleness
+    const threshold = data.stalenessThresholdSeconds ?? 1800;
+    const isStale =
+      !latestRequest ||
+      dayjs().diff(latestRequest.createdAt, 'second') > threshold;
+
+    // 3. If stale, fetch new
+    if (isStale) {
+      const { request } = await this.feedFetcherService.fetchAndSaveResponse(
+        data.url,
+        {
+          lookupDetails: data.lookupKey ? { key: data.lookupKey } : undefined,
+          source: undefined,
+        },
+      );
+
+      await this.partitionedRequestsStoreService.flushInserts([request]);
+
+      // Re-fetch latest
+      latestRequest =
+        await this.partitionedRequestsStoreService.getLatestRequestAnyStatus(
+          lookupKey,
+        );
+    }
+
+    // 4. Map to response
+    if (!latestRequest) {
+      return { requestStatus: 'FETCH_ERROR' as const };
+    }
+
+    const decodedBody = await this.feedFetcherService.decodeResponseContent(
+      latestRequest.response?.content,
+    );
+
+    return this.mapStatusToResponse(
+      latestRequest.status,
+      latestRequest.response,
+      decodedBody,
+    );
+  }
+
+  /**
+   * Maps a request status and response to FetchFeedDetailsDto.
+   * Shared between delivery preview and regular feed request endpoints.
+   */
+  private mapStatusToResponse(
+    status: RequestStatus,
+    response: { textHash?: string | null; statusCode: number } | null,
+    body: string,
+  ): FetchFeedDetailsDto {
+    if (status === RequestStatus.INVALID_SSL_CERTIFICATE) {
+      return { requestStatus: 'INVALID_SSL_CERTIFICATE' as const };
+    }
+
+    if (status === RequestStatus.REFUSED_LARGE_FEED) {
+      return { requestStatus: 'REFUSED_LARGE_FEED' as const };
+    }
+
+    if (status === RequestStatus.FETCH_TIMEOUT) {
+      return { requestStatus: 'FETCH_TIMEOUT' as const };
+    }
+
+    if (status === RequestStatus.FETCH_ERROR || !response) {
+      return { requestStatus: 'FETCH_ERROR' as const };
+    }
+
+    if (status === RequestStatus.OK) {
+      return {
+        requestStatus: 'SUCCESS' as const,
+        response: {
+          hash: response.textHash,
+          body,
+          statusCode: response.statusCode,
+        },
+      };
+    }
+
+    if (status === RequestStatus.PARSE_ERROR) {
+      return {
+        requestStatus: 'PARSE_ERROR' as const,
+        response: { statusCode: response.statusCode },
+      };
+    }
+
+    if (status === RequestStatus.INTERNAL_ERROR) {
+      return { requestStatus: 'INTERNAL_ERROR' as const };
+    }
+
+    if (status === RequestStatus.BAD_STATUS_CODE) {
+      return {
+        requestStatus: 'BAD_STATUS_CODE' as const,
+        response: { statusCode: response.statusCode },
+      };
+    }
+
+    throw new Error(`Unhandled request status: ${status}`);
+  }
+
   private async getLatestRequest(
     data: FetchFeedDto,
   ): Promise<FetchFeedDetailsDto> {
@@ -191,9 +303,7 @@ export class FeedFetcherController {
       };
     }
 
-    const latestRequestStatus = latestRequest.request.status;
-    const latestRequestResponse = latestRequest.request.response;
-
+    // Check for hash match before mapping status
     if (
       data.hashToCompare &&
       latestRequest.request.response?.textHash &&
@@ -204,68 +314,10 @@ export class FeedFetcherController {
       };
     }
 
-    if (latestRequestStatus === RequestStatus.INVALID_SSL_CERTIFICATE) {
-      return {
-        requestStatus: 'INVALID_SSL_CERTIFICATE' as const,
-      };
-    }
-
-    if (latestRequestStatus === RequestStatus.REFUSED_LARGE_FEED) {
-      return {
-        requestStatus: 'REFUSED_LARGE_FEED' as const,
-      };
-    }
-
-    if (latestRequestStatus === RequestStatus.FETCH_TIMEOUT) {
-      return {
-        requestStatus: 'FETCH_TIMEOUT' as const,
-      };
-    }
-
-    if (
-      latestRequestStatus === RequestStatus.FETCH_ERROR ||
-      !latestRequestResponse
-    ) {
-      return {
-        requestStatus: 'FETCH_ERROR' as const,
-      };
-    }
-
-    if (latestRequestStatus === RequestStatus.OK) {
-      return {
-        requestStatus: 'SUCCESS' as const,
-        response: {
-          hash: latestRequestResponse.textHash,
-          body: latestRequest.decodedResponseText as string,
-          statusCode: latestRequestResponse.statusCode,
-        },
-      };
-    }
-
-    if (latestRequestStatus === RequestStatus.PARSE_ERROR) {
-      return {
-        requestStatus: 'PARSE_ERROR' as const,
-        response: {
-          statusCode: latestRequestResponse.statusCode,
-        },
-      };
-    }
-
-    if (latestRequestStatus === RequestStatus.INTERNAL_ERROR) {
-      return {
-        requestStatus: 'INTERNAL_ERROR' as const,
-      };
-    }
-
-    if (latestRequestStatus === RequestStatus.BAD_STATUS_CODE) {
-      return {
-        requestStatus: 'BAD_STATUS_CODE' as const,
-        response: {
-          statusCode: latestRequestResponse.statusCode,
-        },
-      };
-    }
-
-    throw new Error(`Unhandled request status: ${latestRequestStatus}`);
+    return this.mapStatusToResponse(
+      latestRequest.request.status,
+      latestRequest.request.response,
+      latestRequest.decodedResponseText ?? '',
+    );
   }
 }

services/feed-requests/src/feed-fetcher/feed-fetcher.service.ts

@@ -176,6 +176,28 @@ export class FeedFetcherService {
     return { request, decodedResponseText: '' };
   }
 
+  /**
+   * Decode compressed response content from a request.
+   * Used by delivery preview to get the body from any request status.
+   */
+  async decodeResponseContent(
+    compressedContent: string | null | undefined,
+  ): Promise<string> {
+    if (!compressedContent) {
+      return '';
+    }
+
+    try {
+      const decompressed = await inflatePromise(
+        Buffer.from(compressedContent, 'base64'),
+      );
+
+      return decompressed.toString();
+    } catch {
+      return '';
+    }
+  }
+
   async fetchAndSaveResponse(
     url: string,
     options?: {

services/feed-requests/src/partitioned-requests-store/partitioned-requests-store.service.ts

@@ -346,6 +346,32 @@ export default class PartitionedRequestsStoreService {
     return this.mapPartitionedRequestToModel(result);
   }
 
+  /**
+   * Get the latest request for a lookup key, regardless of status or body presence.
+   * Used by delivery preview to check staleness including error states.
+   */
+  async getLatestRequestAnyStatus(lookupKey: string): Promise<Request | null> {
+    const em = this.orm.em.getConnection();
+
+    const [result] = await em.execute(
+      `SELECT req.*, res.content AS response_body_content,
+        res.content_hash AS response_content_hash
+       FROM request_partitioned req
+       LEFT JOIN response_bodies res
+       ON req.response_body_hash_key = res.hash_key
+       WHERE req.lookup_key = ?
+       ORDER BY created_at DESC
+       LIMIT 1`,
+      [lookupKey],
+    );
+
+    if (!result) {
+      return null;
+    }
+
+    return this.mapPartitionedRequestToModel(result);
+  }
+
   private mapPartitionedRequestToModel(result: any) {
     const request: Request = {
       id: result.id,

services/user-feeds-next/.dockerignore

@@ -33,3 +33,4 @@ deploy-api.sh
 .DS_Store
 *.log
 coverage/
+**/*.heapsnapshot

services/user-feeds-next/.gitignore

@@ -33,4 +33,6 @@ report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
 # Finder (MacOS) folder config
 .DS_Store
 
-deploy*.sh
+deploy*.sh
+
+*.heapsnapshot

services/user-feeds-next/src/feed-fetcher/feed-fetcher.ts

@@ -147,3 +147,66 @@ async function handleFetchResponse({
     `Unexpected feed request status in response: ${requestStatus}`
   );
 }
+
+/**
+ * Fetch feed for delivery preview using the dedicated endpoint.
+ * This endpoint checks staleness based on ANY request status (including errors),
+ * preventing duplicate fetches when only error records exist.
+ *
+ * Compatible with fetchFeed signature so it can be passed to fetchAndParseFeed.
+ */
+export async function fetchFeedForDeliveryPreview(
+  url: string,
+  options: {
+    serviceHost: string;
+    stalenessThresholdSeconds?: number;
+    lookupDetails?: FeedRequestLookupDetails | null;
+    // These are accepted for compatibility but not used by this endpoint
+    executeFetch?: boolean;
+    executeFetchIfNotInCache?: boolean;
+    executeFetchIfStale?: boolean;
+    retries?: number;
+    hashToCompare?: string;
+  }
+): Promise<FetchFeedResult> {
+  const serviceHost = options.serviceHost;
+  let response: Response;
+
+  try {
+    const requestBody = {
+      url,
+      lookupKey: options.lookupDetails?.key,
+      stalenessThresholdSeconds: options.stalenessThresholdSeconds,
+    };
+
+    const endpointUrl = serviceHost.endsWith("/")
+      ? `${serviceHost}delivery-preview`
+      : `${serviceHost}/delivery-preview`;
+
+    response = await pRetry(
+      async () =>
+        fetch(endpointUrl, {
+          method: "POST",
+          body: JSON.stringify(requestBody),
+          headers: {
+            "content-type": "application/json",
+            accept: "application/json",
+            "api-key": API_KEY,
+          },
+        }),
+      {
+        retries: 2,
+        randomize: true,
+      }
+    );
+  } catch (err) {
+    throw new FeedRequestNetworkException(
+      `Failed to execute request to feed requests API: ${(err as Error).message}`
+    );
+  }
+
+  return handleFetchResponse({
+    statusCode: response.status,
+    json: response.json.bind(response),
+  });
+}