forked from spatie/crawler
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Feat/convert phpunit tests to pest (spatie#401)
* Install Pest * Convert Tests * Update test command in run-tests.yml * Fix styling * Update Pest.php * Update Log.php Co-authored-by: mansoorkhan96 <[email protected]> Co-authored-by: Freek Van der Herten <[email protected]>
- Loading branch information
1 parent
97acd32
commit 72e5e4b
Showing
11 changed files
with
798 additions
and
880 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -10,3 +10,4 @@ phpunit.xml | |
psalm.xml | ||
vendor | ||
.php-cs-fixer.cache | ||
node_modules |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,112 +1,102 @@ | ||
<?php | ||
|
||
namespace Spatie\Crawler\Test; | ||
|
||
use GuzzleHttp\Psr7\Uri; | ||
use Spatie\Crawler\CrawlQueues\ArrayCrawlQueue; | ||
use Spatie\Crawler\CrawlUrl; | ||
|
||
class ArrayCrawlQueueTest extends TestCase | ||
{ | ||
protected ArrayCrawlQueue $crawlQueue; | ||
beforeEach(function () { | ||
$this->crawlQueue = new ArrayCrawlQueue(); | ||
}); | ||
|
||
protected function setUp(): void | ||
{ | ||
parent::setUp(); | ||
test('a url can be added to crawl queue', function () { | ||
$crawlUrl = createCrawlUrl('https://example.com'); | ||
|
||
$this->crawlQueue = new ArrayCrawlQueue(); | ||
} | ||
$this->crawlQueue->add($crawlUrl); | ||
|
||
/** @test */ | ||
public function an_url_can_be_added() | ||
{ | ||
$crawlUrl = $this->createCrawlUrl('https://example.com'); | ||
$this->crawlQueue->add($crawlUrl); | ||
expect($this->crawlQueue->getPendingUrl()) | ||
->toBe($crawlUrl); | ||
}); | ||
|
||
$this->assertEquals($crawlUrl, $this->crawlQueue->getPendingUrl()); | ||
} | ||
it('can determine if there are pending urls', function () { | ||
expect($this->crawlQueue->hasPendingUrls()) | ||
->toBeFalse(); | ||
|
||
/** @test */ | ||
public function it_can_determine_if_there_are_pending_urls() | ||
{ | ||
$this->assertFalse($this->crawlQueue->hasPendingUrls()); | ||
$this | ||
->crawlQueue | ||
->add(createCrawlUrl('https://example.com')); | ||
|
||
$this->crawlQueue->add($this->createCrawlUrl('https://example.com')); | ||
expect($this->crawlQueue->hasPendingUrls()) | ||
->toBeTrue(); | ||
}); | ||
|
||
$this->assertTrue($this->crawlQueue->hasPendingUrls()); | ||
} | ||
it('can get an url at the specified index', function () { | ||
$url1 = createCrawlUrl('https://example1.com/'); | ||
$url2 = createCrawlUrl('https://example2.com/'); | ||
|
||
/** @test */ | ||
public function it_can_get_an_url_at_the_specified_index() | ||
{ | ||
$url1 = $this->createCrawlUrl('https://example1.com/'); | ||
$url2 = $this->createCrawlUrl('https://example2.com/'); | ||
|
||
$this->crawlQueue->add($url1); | ||
$this->crawlQueue->add($url2); | ||
|
||
$this->assertEquals( | ||
'https://example1.com/', | ||
(string) $this->crawlQueue->getUrlById($url1->getId())->url | ||
); | ||
$this->assertEquals( | ||
'https://example2.com/', | ||
(string) $this->crawlQueue->getUrlById($url2->getId())->url | ||
); | ||
} | ||
$this->crawlQueue->add($url1); | ||
$this->crawlQueue->add($url2); | ||
|
||
/** @test */ | ||
public function it_can_determine_if_has_a_given_url() | ||
{ | ||
$crawlUrl = $this->createCrawlUrl('https://example1.com/'); | ||
$urlInCrawlQueue = (string) $this->crawlQueue->getUrlById($url1->getId())->url; | ||
|
||
$this->assertFalse($this->crawlQueue->has($crawlUrl)); | ||
expect($urlInCrawlQueue) | ||
->toBe('https://example1.com/'); | ||
|
||
$this->crawlQueue->add($crawlUrl); | ||
$urlInCrawlQueue = (string) $this->crawlQueue->getUrlById($url2->getId())->url; | ||
|
||
$this->assertTrue($this->crawlQueue->has($crawlUrl)); | ||
} | ||
expect($urlInCrawlQueue) | ||
->toBe('https://example2.com/'); | ||
}); | ||
|
||
/** @test */ | ||
public function it_can_mark_an_url_as_processed() | ||
{ | ||
$crawlUrl = $this->createCrawlUrl('https://example1.com/'); | ||
it('can determine if has a given url', function () { | ||
$crawlUrl = createCrawlUrl('https://example1.com/'); | ||
|
||
$this->assertFalse($this->crawlQueue->hasAlreadyBeenProcessed($crawlUrl)); | ||
expect($this->crawlQueue->has($crawlUrl)) | ||
->toBeFalse(); | ||
|
||
$this->crawlQueue->add($crawlUrl); | ||
$this->crawlQueue->add($crawlUrl); | ||
|
||
$this->assertFalse($this->crawlQueue->hasAlreadyBeenProcessed($crawlUrl)); | ||
expect($this->crawlQueue->has($crawlUrl)) | ||
->toBeTrue(); | ||
}); | ||
|
||
$this->crawlQueue->markAsProcessed($crawlUrl); | ||
it('can mark a url as processed', function () { | ||
$crawlUrl = createCrawlUrl('https://example1.com/'); | ||
|
||
$this->assertTrue($this->crawlQueue->hasAlreadyBeenProcessed($crawlUrl)); | ||
} | ||
expect($this->crawlQueue->hasAlreadyBeenProcessed($crawlUrl)) | ||
->toBeFalse(); | ||
|
||
/** @test */ | ||
public function it_can_remove_all_processed_urls_from_the_pending_urls() | ||
{ | ||
$crawlUrl1 = $this->createCrawlUrl('https://example1.com/'); | ||
$crawlUrl2 = $this->createCrawlUrl('https://example2.com/'); | ||
$this->crawlQueue->add($crawlUrl); | ||
|
||
$this->crawlQueue | ||
->add($crawlUrl1) | ||
->add($crawlUrl2); | ||
expect($this->crawlQueue->hasAlreadyBeenProcessed($crawlUrl)) | ||
->toBeFalse(); | ||
|
||
$this->crawlQueue->markAsProcessed($crawlUrl1); | ||
$this->crawlQueue->markAsProcessed($crawlUrl); | ||
|
||
$pendingUrlCount = 0; | ||
expect($this->crawlQueue->hasAlreadyBeenProcessed($crawlUrl)) | ||
->toBeTrue(); | ||
}); | ||
|
||
while ($url = $this->crawlQueue->getPendingUrl()) { | ||
$pendingUrlCount++; | ||
$this->crawlQueue->markAsProcessed($url); | ||
} | ||
it('can remove all processed urls from the pending urls', function () { | ||
$crawlUrl1 = createCrawlUrl('https://example1.com/'); | ||
$crawlUrl2 = createCrawlUrl('https://example2.com/'); | ||
|
||
$this->assertEquals(1, $pendingUrlCount); | ||
} | ||
$this->crawlQueue | ||
->add($crawlUrl1) | ||
->add($crawlUrl2); | ||
|
||
$this->crawlQueue->markAsProcessed($crawlUrl1); | ||
|
||
$pendingUrlCount = 0; | ||
|
||
protected function createCrawlUrl(string $url): CrawlUrl | ||
{ | ||
return CrawlUrl::create(new Uri($url)); | ||
while ($url = $this->crawlQueue->getPendingUrl()) { | ||
$pendingUrlCount++; | ||
$this->crawlQueue->markAsProcessed($url); | ||
} | ||
|
||
expect($pendingUrlCount)->toBe(1); | ||
}); | ||
|
||
function createCrawlUrl(string $url): CrawlUrl | ||
{ | ||
return CrawlUrl::create(new Uri($url)); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.