Skip to content

Commit

Permalink
Feat/convert phpunit tests to pest (spatie#401)
Browse files Browse the repository at this point in the history
* Install Pest

* Convert Tests

* Update test command in run-tests.yml

* Fix styling

* Update Pest.php

* Update Log.php

Co-authored-by: mansoorkhan96 <[email protected]>
Co-authored-by: Freek Van der Herten <[email protected]>
  • Loading branch information
3 people authored May 26, 2022
1 parent 97acd32 commit 72e5e4b
Show file tree
Hide file tree
Showing 11 changed files with 798 additions and 880 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/run-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,4 +41,4 @@ jobs:
run: composer update --${{ matrix.dependency-version }} --no-interaction --prefer-source

- name: Execute tests
run: vendor/bin/phpunit
run: vendor/bin/pest
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,4 @@ phpunit.xml
psalm.xml
vendor
.php-cs-fixer.cache
node_modules
1 change: 1 addition & 0 deletions composer.json
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
"symfony/dom-crawler": "^5.2|^6.0"
},
"require-dev": {
"pestphp/pest": "^1.21",
"phpunit/phpunit": "^9.5"
},
"config": {
Expand Down
148 changes: 69 additions & 79 deletions tests/ArrayCrawlQueueTest.php
Original file line number Diff line number Diff line change
@@ -1,112 +1,102 @@
<?php

namespace Spatie\Crawler\Test;

use GuzzleHttp\Psr7\Uri;
use Spatie\Crawler\CrawlQueues\ArrayCrawlQueue;
use Spatie\Crawler\CrawlUrl;

class ArrayCrawlQueueTest extends TestCase
{
protected ArrayCrawlQueue $crawlQueue;
beforeEach(function () {
$this->crawlQueue = new ArrayCrawlQueue();
});

protected function setUp(): void
{
parent::setUp();
test('a url can be added to crawl queue', function () {
$crawlUrl = createCrawlUrl('https://example.com');

$this->crawlQueue = new ArrayCrawlQueue();
}
$this->crawlQueue->add($crawlUrl);

/** @test */
public function an_url_can_be_added()
{
$crawlUrl = $this->createCrawlUrl('https://example.com');
$this->crawlQueue->add($crawlUrl);
expect($this->crawlQueue->getPendingUrl())
->toBe($crawlUrl);
});

$this->assertEquals($crawlUrl, $this->crawlQueue->getPendingUrl());
}
it('can determine if there are pending urls', function () {
expect($this->crawlQueue->hasPendingUrls())
->toBeFalse();

/** @test */
public function it_can_determine_if_there_are_pending_urls()
{
$this->assertFalse($this->crawlQueue->hasPendingUrls());
$this
->crawlQueue
->add(createCrawlUrl('https://example.com'));

$this->crawlQueue->add($this->createCrawlUrl('https://example.com'));
expect($this->crawlQueue->hasPendingUrls())
->toBeTrue();
});

$this->assertTrue($this->crawlQueue->hasPendingUrls());
}
it('can get an url at the specified index', function () {
$url1 = createCrawlUrl('https://example1.com/');
$url2 = createCrawlUrl('https://example2.com/');

/** @test */
public function it_can_get_an_url_at_the_specified_index()
{
$url1 = $this->createCrawlUrl('https://example1.com/');
$url2 = $this->createCrawlUrl('https://example2.com/');

$this->crawlQueue->add($url1);
$this->crawlQueue->add($url2);

$this->assertEquals(
'https://example1.com/',
(string) $this->crawlQueue->getUrlById($url1->getId())->url
);
$this->assertEquals(
'https://example2.com/',
(string) $this->crawlQueue->getUrlById($url2->getId())->url
);
}
$this->crawlQueue->add($url1);
$this->crawlQueue->add($url2);

/** @test */
public function it_can_determine_if_has_a_given_url()
{
$crawlUrl = $this->createCrawlUrl('https://example1.com/');
$urlInCrawlQueue = (string) $this->crawlQueue->getUrlById($url1->getId())->url;

$this->assertFalse($this->crawlQueue->has($crawlUrl));
expect($urlInCrawlQueue)
->toBe('https://example1.com/');

$this->crawlQueue->add($crawlUrl);
$urlInCrawlQueue = (string) $this->crawlQueue->getUrlById($url2->getId())->url;

$this->assertTrue($this->crawlQueue->has($crawlUrl));
}
expect($urlInCrawlQueue)
->toBe('https://example2.com/');
});

/** @test */
public function it_can_mark_an_url_as_processed()
{
$crawlUrl = $this->createCrawlUrl('https://example1.com/');
it('can determine if has a given url', function () {
$crawlUrl = createCrawlUrl('https://example1.com/');

$this->assertFalse($this->crawlQueue->hasAlreadyBeenProcessed($crawlUrl));
expect($this->crawlQueue->has($crawlUrl))
->toBeFalse();

$this->crawlQueue->add($crawlUrl);
$this->crawlQueue->add($crawlUrl);

$this->assertFalse($this->crawlQueue->hasAlreadyBeenProcessed($crawlUrl));
expect($this->crawlQueue->has($crawlUrl))
->toBeTrue();
});

$this->crawlQueue->markAsProcessed($crawlUrl);
it('can mark a url as processed', function () {
$crawlUrl = createCrawlUrl('https://example1.com/');

$this->assertTrue($this->crawlQueue->hasAlreadyBeenProcessed($crawlUrl));
}
expect($this->crawlQueue->hasAlreadyBeenProcessed($crawlUrl))
->toBeFalse();

/** @test */
public function it_can_remove_all_processed_urls_from_the_pending_urls()
{
$crawlUrl1 = $this->createCrawlUrl('https://example1.com/');
$crawlUrl2 = $this->createCrawlUrl('https://example2.com/');
$this->crawlQueue->add($crawlUrl);

$this->crawlQueue
->add($crawlUrl1)
->add($crawlUrl2);
expect($this->crawlQueue->hasAlreadyBeenProcessed($crawlUrl))
->toBeFalse();

$this->crawlQueue->markAsProcessed($crawlUrl1);
$this->crawlQueue->markAsProcessed($crawlUrl);

$pendingUrlCount = 0;
expect($this->crawlQueue->hasAlreadyBeenProcessed($crawlUrl))
->toBeTrue();
});

while ($url = $this->crawlQueue->getPendingUrl()) {
$pendingUrlCount++;
$this->crawlQueue->markAsProcessed($url);
}
it('can remove all processed urls from the pending urls', function () {
$crawlUrl1 = createCrawlUrl('https://example1.com/');
$crawlUrl2 = createCrawlUrl('https://example2.com/');

$this->assertEquals(1, $pendingUrlCount);
}
$this->crawlQueue
->add($crawlUrl1)
->add($crawlUrl2);

$this->crawlQueue->markAsProcessed($crawlUrl1);

$pendingUrlCount = 0;

protected function createCrawlUrl(string $url): CrawlUrl
{
return CrawlUrl::create(new Uri($url));
while ($url = $this->crawlQueue->getPendingUrl()) {
$pendingUrlCount++;
$this->crawlQueue->markAsProcessed($url);
}

expect($pendingUrlCount)->toBe(1);
});

function createCrawlUrl(string $url): CrawlUrl
{
return CrawlUrl::create(new Uri($url));
}
102 changes: 45 additions & 57 deletions tests/CrawlObserverCollectionTest.php
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
<?php

namespace Spatie\Crawler\Test;

use GuzzleHttp\Exception\RequestException;
use GuzzleHttp\Psr7\Request;
use GuzzleHttp\Psr7\Response;
Expand All @@ -12,68 +10,58 @@
use Spatie\Crawler\CrawlObservers\CrawlObserverCollection;
use Spatie\Crawler\CrawlUrl;

class CrawlObserverCollectionTest extends TestCase
{
protected CrawlObserver $crawlObserver;

protected function setUp(): void
{
parent::setUp();

$this->crawlObserver = new class () extends CrawlObserver {
public $crawled = false;
beforeEach(function () {
$this->crawlObserver = new class () extends CrawlObserver {
public $crawled = false;

public $failed = false;
public $failed = false;

public function crawled(
UriInterface $url,
ResponseInterface $response,
?UriInterface $foundOnUrl = null
): void {
$this->crawled = true;
}
public function crawled(
UriInterface $url,
ResponseInterface $response,
?UriInterface $foundOnUrl = null
): void {
$this->crawled = true;
}

public function crawlFailed(
UriInterface $url,
RequestException $requestException,
?UriInterface $foundOnUrl = null
): void {
$this->failed = true;
}
};
}
public function crawlFailed(
UriInterface $url,
RequestException $requestException,
?UriInterface $foundOnUrl = null
): void {
$this->failed = true;
}
};
});

/** @test */
public function it_can_be_fulfilled()
{
$observers = new CrawlObserverCollection([
$this->crawlObserver,
]);
it('can be fulfilled', function () {
$observers = new CrawlObserverCollection([
$this->crawlObserver,
]);

$observers->crawled(
CrawlUrl::create(new Uri('')),
new Response()
);
$observers->crawled(
CrawlUrl::create(new Uri('')),
new Response()
);

$this->assertTrue($this->crawlObserver->crawled);
$this->assertFalse($this->crawlObserver->failed);
}
expect($this->crawlObserver)
->crawled->toBeTrue()
->failed->toBeFalse();
});

/** @test */
public function it_can_fail()
{
$observers = new CrawlObserverCollection([
$this->crawlObserver,
]);
it('can fail', function () {
$observers = new CrawlObserverCollection([
$this->crawlObserver,
]);

$uri = new Uri('');
$uri = new Uri('');

$observers->crawlFailed(
CrawlUrl::create($uri),
new RequestException('', new Request('GET', $uri))
);
$observers->crawlFailed(
CrawlUrl::create(new Uri('')),
new RequestException('', new Request('GET', $uri))
);

$this->assertFalse($this->crawlObserver->crawled);
$this->assertTrue($this->crawlObserver->failed);
}
}
expect($this->crawlObserver)
->crawled->toBeFalse()
->failed->toBeTrue();
});
Loading

0 comments on commit 72e5e4b

Please sign in to comment.