This PHP class allows you to crawl recursively a given html page (or a given html file) and collect some data from it. Simply define the url (or a html file) and a set of xpath expressions which should map with the output data object. The final representation will be a php array which can be easily converted into the json format for further processing.
composer require ixnode/php-web-crawlervendor/bin/php-web-crawler -Vphp-web-crawler 0.1.0 (02-24-2024 14:46:26) - Björn Hempel <bjoern@hempel.li>useIxnode\PhpWebCrawler\Output\Field;
useIxnode\PhpWebCrawler\Source\Raw;
useIxnode\PhpWebCrawler\Value\Text;
useIxnode\PhpWebCrawler\Value\XpathTextNode;
$rawHtml = <<<HTML<html> <head> <title>Test Page</title> </head> <body> <h1>Test Title</h1> <p>Test Paragraph</p> </body></html>HTML;
$html = newRaw(
$rawHtml,
newField('version', newText('1.0.0')),
newField('title', newXpathTextNode('//h1')),
newField('paragraph', newXpathTextNode('//p'))
);
$html->parse()->getJsonStringFormatted();
// See below{
"version": "1.0.0",
"title": "Test Title",
"paragraph": "Test Paragraph"
}useIxnode\PhpWebCrawler\Output\Field;
useIxnode\PhpWebCrawler\Output\Group;
useIxnode\PhpWebCrawler\Source\Raw;
useIxnode\PhpWebCrawler\Value\XpathTextNode;
$rawHtml = <<<HTML<html> <head> <title>Test Page</title> </head> <body> <h1>Test Title</h1> <p class="paragraph-1">Test Paragraph 1</p> <p class="paragraph-2">Test Paragraph 2</p> </body></html>HTML;
$html = newRaw(
$rawHtml,
newField('title', newXpathTextNode('/html/head/title')),
newGroup(
'content',
newGroup(
'header',
newField('h1', newXpathTextNode('/html/body//h1')),
),
newGroup(
'text',
newField('p1', newXpathTextNode('/html/body//p[@class="paragraph-1"]')),
newField('p2', newXpathTextNode('/html/body//p[@class="paragraph-2"]')),
)
)
);
$html->parse()->getJsonStringFormatted();
// See below{
"title": "Test Page",
"content": {
"header": {
"h1": "Test Title"
},
"text": {
"p1": "Test Paragraph 1",
"p2": "Test Paragraph 2"
}
}
}useIxnode\PhpWebCrawler\Output\Field;
useIxnode\PhpWebCrawler\Output\Group;
useIxnode\PhpWebCrawler\Source\Raw;
useIxnode\PhpWebCrawler\Source\XpathSection;
useIxnode\PhpWebCrawler\Value\XpathTextNode;
$rawHtml = <<<HTML<html> <head> <title>Test Page</title> </head> <body> <div class="content"> <h1>Test Title</h1> <p class="paragraph-1">Test Paragraph 1</p> <p class="paragraph-2">Test Paragraph 2</p> </div> </body></html>HTML;
$html = newRaw(
$rawHtml,
newField('title', newXpathTextNode('/html/head/title')),
newGroup(
'content',
newXpathSection(
'/html/body//div[@class="content"]',
newGroup(
'header',
newField('h1', newXpathTextNode('./h1')),
),
newGroup(
'text',
newField('p1', newXpathTextNode('./p[@class="paragraph-1"]')),
newField('p2', newXpathTextNode('./p[@class="paragraph-2"]')),
)
)
)
);
$html->parse()->getJsonStringFormatted();
// See below{
"title": "Test Page",
"content": {
"header": {
"h1": "Test Title"
},
"text": {
"p1": "Test Paragraph 1",
"p2": "Test Paragraph 2"
}
}
}useIxnode\PhpWebCrawler\Output\Field;
useIxnode\PhpWebCrawler\Output\Group;
useIxnode\PhpWebCrawler\Source\Raw;
useIxnode\PhpWebCrawler\Source\XpathSections;
useIxnode\PhpWebCrawler\Value\XpathTextNode;
$rawHtml = <<<HTML<html> <head> <title>Test Page</title> </head> <body> <div class="content"> <h1>Test Title</h1> <p class="paragraph-1">Test Paragraph 1</p> <p class="paragraph-2">Test Paragraph 2</p> <ul> <li>Test Item 1</li> <li>Test Item 2</li> </ul> </div> </body></html>HTML;
$html = newRaw(
$rawHtml,
newField('title', newXpathTextNode('/html/head/title')),
newGroup(
'hits',
newXpathSections(
'/html/body//div[@class="content"]/ul',
newXpathTextNode('./li/text()'),
)
)
);
$html->parse()->getJsonStringFormatted();
// See below{
"title": "Test Page",
"hits": [
[
"Test Item 1",
"Test Item 2"
]
]
}useIxnode\PhpWebCrawler\Output\Field;
useIxnode\PhpWebCrawler\Output\Group;
useIxnode\PhpWebCrawler\Source\Raw;
useIxnode\PhpWebCrawler\Source\XpathSections;
useIxnode\PhpWebCrawler\Value\XpathTextNode;
$rawHtml = <<<HTML<html> <head> <title>Test Page</title> </head> <body> <div class="content"> <h1>Test Title</h1> <p class="paragraph-1">Test Paragraph 1</p> <p class="paragraph-2">Test Paragraph 2</p> <table> <tbody> <tr> <th>Caption 1</th> <td>Cell 1</td> </tr> <tr> <th>Caption 2</th> <td>Cell 2</td> </tr> </tbody> </table> </div> </body></html>HTML;
$html = newRaw(
$rawHtml,
newField('title', newXpathTextNode('/html/head/title')),
newGroup(
'hits',
newXpathSections(
'/html/body//div[@class="content"]/table/tbody/tr',
newField('caption', newXpathTextNode('./th/text()')),
newField('content', newXpathTextNode('./td/text()')),
)
)
);
$html->parse()->getJsonStringFormatted();
// See below{
"title": "Test Page",
"hits": [
{
"caption": "Caption 1",
"content": "Cell 1"
},
{
"caption": "Caption 2",
"content": "Cell 2"
}
]
}- examples/converter.php
- examples/group.php
- examples/section.php
- examples/sections-recursive-url.php
- examples/sections.php
- examples/simple-wiki-page.php
git clone git@github.com:ixnode/php-web-crawler.git &&cd php-web-crawlercomposer installcomposer testThis library is licensed under the MIT License - see the LICENSE.md file for details.