From 3eb8734584283f8ccde2bb294e5662dbc1708991 Mon Sep 17 00:00:00 2001 From: Chris Borg Date: Sat, 13 Mar 2021 17:40:07 +0100 Subject: [PATCH 1/2] adds pagination using page number from feed --- src/models/FeedModel.php | 51 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/src/models/FeedModel.php b/src/models/FeedModel.php index 778cae79..fa72d624 100644 --- a/src/models/FeedModel.php +++ b/src/models/FeedModel.php @@ -219,6 +219,11 @@ public function getFeedMapping($usePrimaryElement = true) */ public function getNextPagination() { + //check if the pagination url provided is just a page number + if ($this->_generateNextPaginationFromPageNumber()) { + return true; + } + if (!$this->paginationUrl || !filter_var($this->paginationUrl, FILTER_VALIDATE_URL)) { return false; } @@ -240,4 +245,48 @@ public function rules() ]; } -} + private function _generateNextPaginationFromPageNumber() { + if (is_numeric($this->paginationUrl)) { + $nextPage = $this->paginationUrl + 1; + if($nextPage > 13) { + return false; + } + + $this->feedUrl = $this->_setPageQueryString($this->feedUrl, "page", $nextPage); + return true; + } + } + + private function _setPageQueryString($url, $param, $value) + { + //remove query string if it already exists + $pieces = parse_url($url); + if (!isset($pieces['query']) || !$pieces['query']) { + return $this->_addQueryString($url, $param, $value); + } + + $query = []; + parse_str($pieces['query'], $query); + if (!isset($query[$param])) { + return $this->_addQueryString($url, $param, $value); + } + + unset($query[$param]); + $pieces['query'] = http_build_query($query); + + $url = http_build_url($pieces); + return $this->_addQueryString($url, $param, $value); + } + + private function _addQueryString($url, $param, $value) + { + $url = preg_replace('/(.*)(?|&)'. $param .'=[^&]+?(&)(.*)/i', '$1$2$4', $url .'&'); + $url = substr($url, 0, -1); + if (strpos($url, '?') === false) { + return ($url .'?'. $param .'='. $value); + } else { + return ($url .'&'. $param .'='. $value); + } + } + +} \ No newline at end of file From 633affc6704a90576ffb76114f4f25686af6adf8 Mon Sep 17 00:00:00 2001 From: Chris Borg Date: Mon, 15 Mar 2021 17:32:06 +0100 Subject: [PATCH 2/2] Refactors next page url generation to the DataType class and uses Craft CMS Url helpers. Includes database migration. --- composer.json | 2 +- src/base/DataType.php | 28 +++++++++- src/controllers/FeedsController.php | 1 + src/migrations/Install.php | 1 + ...164741_migration_for_total_page_number.php | 31 +++++++++++ src/models/FeedModel.php | 54 ++----------------- src/services/Feeds.php | 2 + src/templates/feeds/_element.html | 14 ++++- 8 files changed, 80 insertions(+), 53 deletions(-) create mode 100644 src/migrations/m210313_164741_migration_for_total_page_number.php diff --git a/composer.json b/composer.json index deeb4d11..4b39df86 100644 --- a/composer.json +++ b/composer.json @@ -2,7 +2,7 @@ "name": "craftcms/feed-me", "description": "Import content from XML, RSS, CSV or JSON feeds into entries, categories, Craft Commerce products, and more.", "type": "craft-plugin", - "version": "4.3.6", + "version": "4.3.7", "keywords": [ "craft", "cms", diff --git a/src/base/DataType.php b/src/base/DataType.php index 9b303dfa..f82c1b04 100644 --- a/src/base/DataType.php +++ b/src/base/DataType.php @@ -41,10 +41,16 @@ public function setupPaginationUrl($array, $feed) if (!$feed->paginationNode) { return; } - // Find the URL value in the feed $flatten = Hash::flatten($array, '/'); $url = Hash::get($flatten, $feed->paginationNode); + $totalPages = Hash::get($flatten, $feed->paginationTotalNode); + + //check if the pagination url provided is just a page number + $pagedUrl = $this->_generateNextPaginationFromPageNumber($feed->feedUrl, $url, $totalPages); + if($pagedUrl) { + $url = $pagedUrl; + } // if the feed provides a root relative URL, make it whole again based on the feed. if ($url && UrlHelper::isRootRelativeUrl($url)) { @@ -55,4 +61,24 @@ public function setupPaginationUrl($array, $feed) $feed->paginationUrl = $url; } + /** + * Generates the next page url if the next page provided in the field is a page number + * instead of a url. In this case, the total pages field needs to be provided as well. + * @param $feedUrl + * @param $url + * @param $totalPages + */ + private function _generateNextPaginationFromPageNumber($feedUrl, $url, $totalPages) { + if (is_numeric($url) && is_numeric($totalPages)) { + $nextPage = $url + 1; + if($nextPage > $totalPages) { + return null; + } + + $feedUrl = UrlHelper::removeParam($feedUrl, "page"); + return UrlHelper::urlWithParams($feedUrl, array("page" => $nextPage)); + } + return null; + } + } diff --git a/src/controllers/FeedsController.php b/src/controllers/FeedsController.php index 95ec7a07..c07eb7c1 100644 --- a/src/controllers/FeedsController.php +++ b/src/controllers/FeedsController.php @@ -400,6 +400,7 @@ private function _getModelFromPost() $feed->singleton = $request->getBodyParam('singleton', $feed->singleton); $feed->duplicateHandle = $request->getBodyParam('duplicateHandle', $feed->duplicateHandle); $feed->paginationNode = $request->getBodyParam('paginationNode', $feed->paginationNode); + $feed->paginationTotalNode = $request->getBodyParam('paginationTotalNode', $feed->paginationTotalNode); $feed->passkey = $request->getBodyParam('passkey', $feed->passkey); $feed->backup = (bool)$request->getBodyParam('backup', $feed->backup); diff --git a/src/migrations/Install.php b/src/migrations/Install.php index e0074ea6..03e2c14a 100644 --- a/src/migrations/Install.php +++ b/src/migrations/Install.php @@ -41,6 +41,7 @@ protected function createTables() 'singleton' => $this->boolean()->notNull()->defaultValue(false), 'duplicateHandle' => $this->text(), 'paginationNode' => $this->text(), + 'paginationTotalNode' => $this->text(), 'fieldMapping' => $this->text(), 'fieldUnique' => $this->text(), 'passkey' => $this->string()->notNull(), diff --git a/src/migrations/m210313_164741_migration_for_total_page_number.php b/src/migrations/m210313_164741_migration_for_total_page_number.php new file mode 100644 index 00000000..a7d04f86 --- /dev/null +++ b/src/migrations/m210313_164741_migration_for_total_page_number.php @@ -0,0 +1,31 @@ +db->columnExists('{{%feedme_feeds}}', 'paginationTotalNode')) { + $this->addColumn('{{%feedme_feeds}}', 'paginationTotalNode', $this->text()->after('duplicateHandle')); + } + } + + /** + * @inheritdoc + */ + public function safeDown() + { + echo "m210313_164741_migration_for_total_page_number cannot be reverted.\n"; + return false; + } +} diff --git a/src/models/FeedModel.php b/src/models/FeedModel.php index fa72d624..72882100 100644 --- a/src/models/FeedModel.php +++ b/src/models/FeedModel.php @@ -83,6 +83,11 @@ class FeedModel extends Model */ public $paginationNode; + /** + * @var + */ + public $paginationTotalNode; + /** * @var */ @@ -219,11 +224,6 @@ public function getFeedMapping($usePrimaryElement = true) */ public function getNextPagination() { - //check if the pagination url provided is just a page number - if ($this->_generateNextPaginationFromPageNumber()) { - return true; - } - if (!$this->paginationUrl || !filter_var($this->paginationUrl, FILTER_VALIDATE_URL)) { return false; } @@ -245,48 +245,4 @@ public function rules() ]; } - private function _generateNextPaginationFromPageNumber() { - if (is_numeric($this->paginationUrl)) { - $nextPage = $this->paginationUrl + 1; - if($nextPage > 13) { - return false; - } - - $this->feedUrl = $this->_setPageQueryString($this->feedUrl, "page", $nextPage); - return true; - } - } - - private function _setPageQueryString($url, $param, $value) - { - //remove query string if it already exists - $pieces = parse_url($url); - if (!isset($pieces['query']) || !$pieces['query']) { - return $this->_addQueryString($url, $param, $value); - } - - $query = []; - parse_str($pieces['query'], $query); - if (!isset($query[$param])) { - return $this->_addQueryString($url, $param, $value); - } - - unset($query[$param]); - $pieces['query'] = http_build_query($query); - - $url = http_build_url($pieces); - return $this->_addQueryString($url, $param, $value); - } - - private function _addQueryString($url, $param, $value) - { - $url = preg_replace('/(.*)(?|&)'. $param .'=[^&]+?(&)(.*)/i', '$1$2$4', $url .'&'); - $url = substr($url, 0, -1); - if (strpos($url, '?') === false) { - return ($url .'?'. $param .'='. $value); - } else { - return ($url .'&'. $param .'='. $value); - } - } - } \ No newline at end of file diff --git a/src/services/Feeds.php b/src/services/Feeds.php index c1b46b18..384c724c 100644 --- a/src/services/Feeds.php +++ b/src/services/Feeds.php @@ -123,6 +123,7 @@ public function saveFeed(FeedModel $model, bool $runValidation = true): bool $record->singleton = (bool)$model->singleton; $record->duplicateHandle = $model->duplicateHandle; $record->paginationNode = $model->paginationNode; + $record->paginationTotalNode = $model->paginationTotalNode; $record->passkey = $model->passkey; $record->backup = $model->backup; @@ -252,6 +253,7 @@ private function _getQuery() 'singleton', 'duplicateHandle', 'paginationNode', + 'paginationTotalNode', 'fieldMapping', 'fieldUnique', 'passkey', diff --git a/src/templates/feeds/_element.html b/src/templates/feeds/_element.html index 07ec52b7..dc3bc259 100644 --- a/src/templates/feeds/_element.html +++ b/src/templates/feeds/_element.html @@ -60,8 +60,8 @@ {% endfor %} {{ forms.selectField({ - label: "Pagination URL"|t('feed-me'), - instructions: 'If your feed is paginated, select the next page’s URL.'|t('feed-me'), + label: "Pagination URL or Page Number"|t('feed-me'), + instructions: 'If your feed is paginated, select the next page’s URL, or the current page number.'|t('feed-me'), id: 'paginationNode', name: 'paginationNode', value: feed.paginationNode, @@ -69,6 +69,16 @@ errors: feed.getErrors('paginationNode'), }) }} + {{ forms.selectField({ + label: "Pagination Total Pages"|t('feed-me'), + instructions: 'If your feed is paginated using page numbers, select the field containing the total pages. This is required if your feed uses page numbers.'|t('feed-me'), + id: 'paginationTotalNode', + name: 'paginationTotalNode', + value: feed.paginationTotalNode, + options: parsedFeedData, + errors: feed.getErrors('paginationTotalNode'), + }) }} + {% else %}