clean: リトライ処理きれいに

komura-c · May 1, 2022 · a697ba1 · a697ba1
1 parent 3c831e0
commit a697ba1
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 36 deletions.
diff --git a/src/feed/utils/common-util.ts b/src/feed/utils/common-util.ts
@@ -1,5 +1,6 @@
 import * as v8 from 'v8';
 import * as crypto from 'crypto';
+import retry from 'async-retry';
 
 export const objectDeepCopy = <T>(data: T): T => {
   // TODO: Node.js 17 以上にしたら structuredClone 使う
@@ -34,3 +35,10 @@ export const isValidHttpUrl = (url: string) => {
 
   return urlObject.protocol === 'http:' || urlObject.protocol === 'https:';
 };
+
+export const backoff = async <A>(retrier: retry.RetryFunction<A>): Promise<A> => {
+  return await retry(retrier, {
+    retries: 3,
+    factor: 2,
+  });
+};
diff --git a/src/feed/utils/feed-crawler.ts b/src/feed/utils/feed-crawler.ts
@@ -4,8 +4,7 @@ import { FeedInfo } from '../../resources/feed-info-list';
 import dayjs from 'dayjs';
 import axios from 'axios';
 import { URL } from 'url';
-import retry from 'async-retry';
-import { isValidHttpUrl, objectDeepCopy, urlRemoveQueryParams } from './common-util';
+import { backoff, isValidHttpUrl, objectDeepCopy, urlRemoveQueryParams } from './common-util';
 import { logger } from './logger';
 import constants from '../../common/constants';
 const ogs = require('open-graph-scraper');
@@ -67,14 +66,9 @@ export class FeedCrawler {
         logger.trace(error);
       })
       .process(async (feedInfo) => {
-        const feed = await retry(
-          async () => {
-            return (await this.rssParser.parseURL(feedInfo.url)) as CustomRssParserFeed;
-          },
-          {
-            retries: 3,
-          },
-        );
+        const feed = await backoff(async () => {
+          return this.rssParser.parseURL(feedInfo.url) as Promise<CustomRssParserFeed>;
+        });
         const postProcessedFeed = FeedCrawler.postProcessFeed(feedInfo, feed);
         feeds.push(postProcessedFeed);
         logger.info('[fetch-feed] fetched', `${fetchProcessCounter++}/${feedInfoListLength}`, feedInfo.label);
@@ -220,14 +214,9 @@ export class FeedCrawler {
         logger.trace(error);
       })
       .process(async (feedItem) => {
-        const ogsResult = await retry(
-          async () => {
-            return await FeedCrawler.fetchOgsResult(feedItem.link);
-          },
-          {
-            retries: 3,
-          },
-        );
+        const ogsResult = await backoff(async () => {
+          return FeedCrawler.fetchOgsResult(feedItem.link);
+        });
         feedItemOgsResultMap.set(feedItem.link, ogsResult);
         logger.info('[fetch-feed-item-ogp] fetched', `${fetchProcessCounter++}/${feedItemsLength}`, feedItem.title);
       });
@@ -247,14 +236,9 @@ export class FeedCrawler {
         logger.trace(error);
       })
       .process(async (feed) => {
-        const ogsResult = await retry(
-          async () => {
-            return await FeedCrawler.fetchOgsResult(feed.link);
-          },
-          {
-            retries: 3,
-          },
-        );
+        const ogsResult = await backoff(async () => {
+          return FeedCrawler.fetchOgsResult(feed.link);
+        });
         feedOgsResultMap.set(feed.link, ogsResult);
         logger.info('[fetch-feed-ogp] fetched', `${fetchProcessCounter++}/${feedsLength}`, feed.title);
       });

diff --git a/tests/feed-info-list.test.ts b/tests/feed-info-list.test.ts
@@ -1,8 +1,8 @@
 import { FEED_INFO_LIST, FeedInfo } from '../src/resources/feed-info-list';
 import { FeedCrawler } from '../src/feed/utils/feed-crawler';
 import { describe, it, expect } from 'vitest';
+import { backoff } from '../src/feed/utils/common-util';
 const RssParser = require('rss-parser');
-const retry = require('async-retry');
 
 const rssParser = new RssParser();
 
@@ -18,17 +18,13 @@ describe('FEED_INFO_LIST', () => {
 // フィード取得テスト
 describe('フィードが取得可能', () => {
   FEED_INFO_LIST.map((feedInfo: FeedInfo) => {
+    const testTitle = `${feedInfo.label} / ${feedInfo.url}`;
     it.concurrent(
-      `${feedInfo.label} / ${feedInfo.url}`,
+      testTitle,
       async () => {
-        const feed = await retry(
-          async () => {
-            return rssParser.parseURL(feedInfo.url);
-          },
-          {
-            retries: 3,
-          },
-        );
+        const feed = await backoff(async () => {
+          return rssParser.parseURL(feedInfo.url);
+        });
         expect(feed.items.length).toBeGreaterThanOrEqual(0);
       },
       60 * 1000,