php curl multithread

[paxnWo]

<?php

function curl($nodes, $referer)
{

set_time_limit(0);
if(!$referer){
$referer = $nodes[0];
}
$node_count = count($nodes);

$curl_arr = array();
$master = curl_multi_init();

for($i = 0; $i < $node_count; $i++)
{
$curl_arr[$i] = curl_init($nodes[$i]);
curl_setopt($curl_arr[$i],CURLOPT_FRESH_CONNECT,true);
curl_setopt($curl_arr[$i],CURLOPT_CONNECTTIMEOUT,10);
curl_setopt($curl_arr[$i],CURLOPT_RETURNTRANSFER,true);
curl_setopt($curl_arr[$i],CURLOPT_REFERER,$referer);
curl_setopt($curl_arr[$i],CURLOPT_TIMEOUT,30);

curl_multi_add_handle($master, $curl_arr[$i]);
}
$finalresult = array();
$returnedOrder = array();
do{
curl_multi_exec($master, $running);
$info = curl_multi_info_read($master);
if($info['handle']){
$finalresult[] = curl_multi_getcontent($info['handle']);
$returnedOrder[] = array_search($info['handle'], $curl_arr, true);
curl_multi_remove_handle($master, $info['handle']);
curl_close($curl_arr[end($returnedOrder)]);
}

$previousActive = $running;
}while($running > 0);

return array_combine($returnedOrder, $finalresult);
curl_multi_close($master);

}

$urls = array( 'link1', 'link2' );
curl($urls, null);

?>

argumentul $nodes al functiei trebuie sa fie un array.

functia am gasit-o pe net si am rescris-o.

cu curl fara mulithread am prelucrat 100 de pagini in 32 de secunde.

cu curl cu multithread am facut-o in doar 2.

preluata de aici : Rusty Razor Blade Blog Archive Executing multiple curl requests in parallel with PHP and curl_multi_exec

am de prelucrat 2.400.000 pagini, caut dedicat pe care sa il omor. am nevoie doar o saptamana. cumpar.

Edited August 21, 2010 by paxnWo