web crawler(Php)

[JOHNYS22031990]

εχω βρει αυτο το web crawler και προσπαθω να τον καταλαβω ο κωδικας ειναι αυτοσ

<?

/**

*

*
* About class:
*  WebSpider    -  constructor set teh domain & url to map it 
*  
*   processTagInPageData() 	- it process the anchor tag & frame tag as googlebot does
*   fetchURLPageData() 		- it returns the html page content for the given URL
* 	isURLExists()			- it checks wheather the given url is added in DB  
*   displayDomainRecords    - it displays teh records  from DB    
* 	StoreUniqueURL 			- it stores the unique url in DB 
* 	processSpecificTagbyType-  it parse each & every tag & truncate the parsed tag from the string 
*/



ini_set("display_errors",1); 

$con=mysqli_connect("localhost","root","");//sindesi
// Check connection
if (mysqli_connect_errno()) {
  echo "Failed to connect to MySQL: " . mysqli_connect_error();
}

// Create database
$sql="CREATE DATABASE crawler";
if (mysqli_query($con,$sql)) {
  echo "Database my_db created successfully";
} else {
  echo "Error creating database: " . mysqli_error($con);
}


$sql1=CREATE TABLE `spider` (
  `id` bigint(20) NOT NULL auto_increment,
  `domain` varchar(150) NOT NULL,
  `url` varchar(2000) NOT NULL,
  `parentid` int(11) NOT NULL,
  `visitflag` int(11) NOT NULL,
  `type` varchar(20) NOT NULL,
  `level` mediumint(9) NOT NULL,
  PRIMARY KEY  (`id`)
); 
if (mysqli_query($con,$sql1)) {
  echo " table1 created successfully";
} else {
  echo "Error creating table: " . mysqli_error($con);
}



class WebSpider {

var $mMaxDepth=5;
var $mDomain="localhost";
var $mDBHost="localhost";
var $mDBUserName="root";
var $mDBPassword="";
var $mDBDatabase="crawler";
//var $mURLPageData;
var $mURL=$_POST["url"];


	function WebSpider($mDomain, $mDepth,$mURL) { 
	
		$this->mDomain		= $mDomain;
		$this->mMaxDepth	= $mDepth;
		$this->mURL			= $mURL;
					
	}
	
	function isURLExists($mDomain, $mURL) {  

		mysqli_connect($this->mDBHost,$this->mDBUserName,$this->mDBPassword);
		mysqli_select_db($this->mDBDatabase);  
		
		$vSQL  = "SELECT count( id ) AS cnt FROM spider
					WHERE  domain = '$mDomain' and url = '$mURL'";
		$rs = mysqli_query($vSQL); 
		$oRecord = mysqli_fetch_assoc($rs);
		return $oRecord['cnt'];
		
    }

	function displayDomainRecords($mDomain){
	
	   	mysqli_connect($this->mDBHost,$this->mDBUserName,$this->mDBPassword);
		mysqli_select_db($this->mDBDatabase);  		
		
		$vSQL = "select count(id) as cnt   from  spider where domain = '$mDomain' ";
		$rsURLList = mysqli_query($vSQL);
		$vCnt = mysqli_fetch_assoc($rsURLList);
		
			
		$vSQL = "select *   from  spider where domain = '$mDomain' order by id asc ";
		$rsURLList = mysqli_query($vSQL);
		print "<strong>Domain</strong> : ".$mDomain ." <strong>Total URL</strong> :".$vCnt['cnt'];
		?>
		<table width='80%'  cellspacing='2' cellpadding='2' border="1">
		  <tr>
			<td><strong>URL</strong></td>
			<td><strong>Type</strong></td>
		  </tr> 
		<?php 
		
		while($Rec = mysqli_fetch_assoc($rsURLList)) { ?>			
   		  <tr>
			<td><?php echo $Rec['url'] ?></td>
			<td><?php echo htmlspecialchars($Rec['type'] )?></td>

		  </tr>
     	<? }  ?> 
         </table>
        
        <?php 
		

	
	}
	
	function StoreUniqueURL($mDomain, $mURL, $mParentId=0, $mLevel,$mType){
	
		mysqli_connect($this->mDBHost,$this->mDBUserName,$this->mDBPassword);
		mysqli_select_db($this->mDBDatabase);  		 
	
		$pmURL = mysqli_real_escape_string($mURL); 
		if($this->isURLExists($mDomain,$mURL)==0) {  	
			$vURLSQL = " INSERT INTO `spider` ( domain, `url` , `parentid` , `visitflag` , `type` , `level` )
							VALUES ('$mDomain' , '$mURL', '$mParentId', '0', '$mType', '$mLevel' )";				
			mysqli_query($vURLSQL); 	
		}
		
	}


	function fetchLinkfromTag($mData, $mTagName, $mAtributeName){ 
	
		$vPos = strpos($mData, $mTagName);
		if($vPos === false){
			return false; // if no link found stop search 
		}
		$vPos += strlen($vStr); 	
		$vSubStr = substr($mData,$vPos);
		
		$vHrefPos = strpos($vSubStr, $mAtributeName);
		$vSubStr = substr($vSubStr,  $vHrefPos);
	
		
		$url = explode('"',$vSubStr);	
		return array("url"=>$url[1],"str"=>$vSubStr);
		
	}




	function fetchURLPageData($vURL) { 

		$rCurlRes = curl_init();
		curl_setopt($rCurlRes, CURLOPT_URL,$vURL);		
		curl_setopt($rCurlRes, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.13)');
		curl_setopt($rCurlRes, CURLOPT_REFERER, $this->mDomain);
		curl_setopt($rCurlRes, CURLOPT_AUTOREFERER, true); 	  
		curl_setopt($rCurlRes, CURLOPT_HEADER, 0); // set to 0 to eliminate header info from response
		curl_setopt($rCurlRes, CURLOPT_RETURNTRANSFER, 1); // Returns response data instead of TRUE(1	
		$res = curl_exec($rCurlRes);
		return $res;
	}
	
   function ProcessSpiderInit() { 
         
	$this->StoreUniqueURL($this->mDomain, $this->mURL, 0,1,'index');  
	
   }
     

   function processSpecificTagbyType($mData, $mTagName, $mAttribute) {
      
    do {    
			$aResult = $this->fetchLinkfromTag($mData,$mTagName, $mAttribute); 			
			
			$vURL 	= $aResult['url'];
			$pmData = $aResult['str']; 			
			if($pmData) {
				$this->StoreUniqueURL($this->mDomain, $vURL , 1,  1, $mTagName);  
			}
			$vIndex++;    			
			
	 }while($mData);	 
   
   }   


   
   function processTagInPageData($mData) {  
   
   	   $this->processSpecificTagbyType($mData,'<a',"href=");  
       $this->processSpecificTagbyType($mData,'<frame', "src=");       
  
   }
   
   
   function fetchURLDataandParseURL() { 

	  $vData   = $this->fetchURLPageData($this->mURL);  	
   	  $this->processTagInPageData($vData);  	  
  }		



?>

αυτο που δεν μπορω να καταλαβω ειναι αυτο το κομματι

class WebSpider {

var $mMaxDepth;
var $mDomain;
var $mDBHost;
var $mDBUserName;
var $mDBPassword;
var $mDBDatabase;
var $mURLPageData;
var $mURl;

δεν πρεπει να παιρνουν καποιες τιμες

[geomagas]

αυτο που δεν μπορω να καταλαβω ειναι αυτο το κομματι

class WebSpider {

var $mMaxDepth;
var $mDomain;
var $mDBHost;
var $mDBUserName;
var $mDBPassword;
var $mDBDatabase;
var $mURLPageData;
var $mURl;

δεν πρεπει να παιρνουν καποιες τιμες

 

Κάποια από αυτά παίρνουν τιμές στον constructor, κάποια άλλα χρησιμοποιούνται αλλού...

Παραπάνω πάντως, τα αρχικοποιείς. Αυτός να φανταστώ είναι ο original κώδικας και στο πρώτο listing τον έχεις αλλάξει;

 

Φαντάζομαι επίσης ότι, οτιδήποτε είναι πάνω από τη δήλωση της class είναι δικές σου προσθήκες, έτσι;